def compare_spaces(self):
self._compute_graph_laplacians()
sent_eigenvalues = torch.eig(self.sent_laplacian, eigenvectors=False)[0]
sent_sorted_eigenvalues = torch.sort(sent_eigenvalues[:, 0], -1, True)[0] # .numpy()
sent_eignvalue_target = 0.9 * torch.sum(sent_sorted_eigenvalues).item()
sent_cumsum = torch.cumsum(sent_sorted_eigenvalues, dim=0)
sent_min_k = torch.min(torch.where(sent_cumsum > sent_eignvalue_target)[0]).item()
#sev = list(chain.from_iterable(sent_eigenvalues.numpy()))
#sent_min_k = 0
#for k in range(len(sent_sorted_eigenvalues)):
# val = sent_sorted_eigenvalues[0:k].sum()
# check = [1 if val > x else 0 for x in sev]
# if sum(check) < .9 * len(sent_sorted_eigenvalues):
# sent_min_k = k
print('Sent min k: {k}'.format(k=sent_min_k))
node_eigenvalues = torch.eig(self.node_laplacian, eigenvectors=False)[0]
node_sorted_eigenvalues = torch.sort(node_eigenvalues[:, 0], -1, True)[0] # .numpy()
node_eignvalue_target = 0.9 * torch.sum(node_sorted_eigenvalues).item()
node_cumsum = torch.cumsum(node_sorted_eigenvalues, dim=0)
node_min_k = torch.min(torch.where(node_cumsum > node_eignvalue_target)[0]).item()
#nev = list(chain.from_iterable(node_eigenvalues.numpy()))
#node_min_k = 0
#for k in range(len(node_eigenvalues)):
# val = node_sorted_eigenvalues[0:k].sum()
# check = [1 if val > x else 0 for x in nev]
# if sum(check) < .9 * len(node_sorted_eigenvalues):
# node_min_k = k
print('Node min k: {k}'.format(k=node_min_k))
final_k = min(node_min_k, sent_min_k)
node_k = node_sorted_eigenvalues[0: final_k]
sent_k = sent_sorted_eigenvalues[0: final_k]
delta = self.sum_of_squares(node_k, sent_k).item()
return delta
def compute_fisher(self, action_log_probs):
# compute logprob grad for each action
grad_log_probs = []
self.F.zero_()
for i in range(len(action_log_probs)):
# For each log pi(a | s) get the gradient wrt theta
self.zero_grad()
action_log_probs[i].backward(retain_graph=True)
grad = self.get_flattened_grad()
grad_log_probs.append(grad)
# Build matrix of gradients (nparams x nsamples)
X = torch.cat(grad_log_probs, dim=1)
self.F = torch.mm(X, torch.transpose(X, 0, 1))
self.F = self.F / float(len(action_log_probs))
# check if Fisher PSD, if not add eps * eye
min_eig = torch.min(torch.eig(self.F)[0])
trys = 0
while min_eig < 0:
trys += 1
print("Fisher eig < 0: ", float(min_eig), file=sys.stderr)
self.F = self.F + 1e-5 * torch.eye(self.F.shape[0])
min_eig = torch.min(torch.eig(self.F)[0])
if trys > 100:
print("Fisher numerically unstable, matrix is neg def.",
file=sys.stderr)
sys.exit()
def _update_inv(self, m):
"""Do eigen decomposition for computing inverse of the ~ fisher.
:param m: The layer
:return: no returns.
"""
if self.torch_symeig == 'true':
eps = 1e-10 # for numerical stability
self.d_a[m], self.Q_a[m] = torch.symeig(self.m_aa[m],
eigenvectors=True)
self.d_g[m], self.Q_g[m] = torch.symeig(self.m_gg[m],
eigenvectors=True)
self.d_a[m].mul_((self.d_a[m] > eps).float())
self.d_g[m].mul_((self.d_g[m] > eps).float())
# if self.steps != 0:
self.S_l[m] = self.d_g[m].unsqueeze(1) @ self.d_a[m].unsqueeze(0)
else:
d_a, self.Q_a[m] = torch.eig(self.m_aa[m], eigenvectors=True)
d_g, self.Q_g[m] = torch.eig(self.m_gg[m], eigenvectors=True)
self.d_a[m] = d_a[:, 0]
self.d_g[m] = d_g[:, 0]
self.d_a[m].mul_((self.d_a[m] > eps).float())
self.d_g[m].mul_((self.d_g[m] > eps).float())
self.S_l[m] = self.d_g[m].unsqueeze(1) @ self.d_a[m].unsqueeze(0)
def hottaBasisVectors(X,sub_dim):
d,n = X.shape
if d < n:
C = torch.mm(X, X.transpose(0,1))
matrank = torch.matrix_rank(C)
tmp_val, tmp_vec = torch.eig(C, eigenvectors = True)
value, index = torch.sort(tmp_val,descending = True)
eig_vec = tmp_vec[:,index[:matrank]]
eig_val = value[:matrank]
eig_vec = eig_vec[:,:sub_dim]
eig_val = value[:sub_dim]
else:
C = torch.mm(X.transpose(0,1), X)
matrank = torch.matrix_rank(C)
tmp_val, tmp_vec = torch.eig(C, eigenvectors = True)
# second column is zero if the eig vals are real
tmp_val = tmp_val[:,0]
value, index = torch.sort(tmp_val,descending = True)
# tmp_vec = tmp_vec[:,index[:matrank]]
# eig_vec = torch.mm(X, tmp_vec)
eig_vec = torch.zeros((X.shape[0],matrank))
for i in range(matrank):
eig_vec[:,i] = (X.mv(tmp_vec[:,index[i]])).div((value[i]).sqrt())
eig_vec = normalize_dataset(eig_vec[:,:sub_dim])
eig_val = value[:sub_dim]
return eig_vec, eig_val
def qt_cal(x, y):
input_outer_product = torch.outer(x, x)
output_outer_product = torch.outer(y, y)
(input_evals, input_evecs) = torch.eig(input_outer_product,
eigenvectors=True)
(output_evals, output_evecs) = torch.eig(output_outer_product,
eigenvectors=True)
def forward(self, x_train, y_train, x_test=None):
# See the autograd section for explanation of what happens here.
n = x_train.size(0)
#dm = torch.Tensor([[1,0,0],[0,1,0],[0,1,0],[0,0,1]])
SIGMA = torch.diag(self.sigmaentr.pow(2).view(
-1)) #dm.mm(self.sigmaentr.pow(2).view(3,1)).view(2,2)
trOnes = torch.ones(1, n)
xext = torch.cat((trOnes, x_train.t()), 0)
xSx = xext.t().mm(SIGMA).mm(xext)
xSxd = torch.diag(xSx).view(n, 1)
kyy = (2.0 / math.pi) * torch.asin(2.0 * xSx / torch.sqrt(
(1 + 2.0 * xSxd).mm(1 + 2.0 * xSxd.t())))
kyy = 0.5 * (kyy + kyy.t()) + self.sigma_n.pow(2) * torch.eye(n)
add = torch.eig(kyy)[0][:, 0].min().detach()
while add < 0:
kyy = kyy - 2 * add * torch.eye(n)
add = torch.eig(kyy)[0][:, 0].min().detach()
c = torch.cholesky(kyy, upper=True)
# v = torch.potrs(y_train, c, upper=True)
v, _ = torch.gesv(y_train, kyy)
if x_test is None:
out = (c, v)
if x_test is not None:
nt = x_test.size(0)
teOnes = torch.ones(1, nt)
xexte = torch.cat((teOnes, x_test.t()), 0)
xSx_m = xext.t().mm(SIGMA).mm(xexte)
xSx_te_d = torch.sum(xexte.t() * (SIGMA.mm(xexte)).t(),
dim=1).view(nt, 1)
kyf = (2.0 / math.pi) * torch.asin(2.0 * xSx_m / torch.sqrt(
(1 + 2.0 * xSxd).mm(1 + 2.0 * xSx_te_d.t())))
# solve
f_test = kyf.t().mm(v)
# tmp = torch.potrs(kfy.t(), c, upper=True)
# tmp = torch.sum(kfy * tmp.t(), dim=1)
# #cov_f = something - tmp
out = f_test #, cov_f)
return out
def forward(self, s_inputs, s_outputs, t_input):
num_src_domains = len(s_inputs)
train_losses, discs = [], []
Phi_t = torch.cat([t_input, torch.ones_like(t_input)], dim=1)
M_t = torch.mm(Phi_t.transpose(0, 1), Phi_t)
for i in range(num_src_domains):
prediction = self.regress_net(s_inputs[i])
train_loss = torch.mean((prediction - s_outputs[i]) ** 2)
train_losses.append(train_loss)
Phi_s = torch.cat([s_inputs[i], torch.ones_like(s_inputs[i])], dim=1)
M_s = torch.mm(Phi_s.transpose(0, 1), Phi_s)
disc = torch.max(torch.norm(torch.eig(M_t - M_s)[0], dim=1))
discs.append(disc)
train_losses = torch.stack(train_losses)
discs = torch.stack(discs)
g = self.gamma * (train_losses + self.mu * discs)
alpha = self.proj(g)
loss = torch.dot(g, alpha) + torch.norm(alpha)
alpha = alpha.detach().cpu().numpy()
return loss, alpha
def spectral_normalization(W):
'''
return net/||net||_spectral
'''
eigenvalues = torch.eig(W.mm(W))
spectral_norm = torch.max(eigenvalues)**0.5
return spectral_norm
def calculate_frechet_distance(mu1: torch.Tensor,
sigma1: torch.Tensor,
mu2: torch.Tensor,
sigma2: torch.Tensor,
eps: float = 1e-8) -> torch.Tensor:
"""Numpy implementation of the Frechet Distance.
The Frechet distance between two multivariate Gaussians X_1 ~ N(mu_1, C_1)
and X_2 ~ N(mu_2, C_2) is
d^2 = ||mu_1 - mu_2||^2 + Tr(C_1 + C_2 - 2*sqrt(C_1*C_2)).
Stable version by Dougal J. Sutherland.
Params:
-- mu1 : Numpy array containing the activations of a layer of the
inception net (like returned by the function 'get_predictions')
for generated samples.
-- mu2 : The sample mean over activations, precalculated on an
representative data set.
-- sigma1: The covariance matrix over activations for generated samples.
-- sigma2: The covariance matrix over activations, precalculated on an
representative data set.
Returns:
-- : The Frechet Distance.
"""
diff = mu1 - mu2
offset = eps * torch.eye(*sigma1.shape, device=sigma1.device)
eigenvals, _ = torch.eig((sigma1 + offset) @ (sigma2 + offset),
eigenvectors=False)
tr_covmean = torch.sum(torch.sqrt(torch.abs(eigenvals[:, 0])))
return (diff.dot(diff) + torch.trace(sigma1) + torch.trace(sigma2) -
2 * tr_covmean)
def hessian_max_min_eig(grads, model):
hess = hessian(grads, model) #.to(device)
eigvalues = torch.eig(hess)[0]
real_eigvalues = [
real for real, img in eigvalues if torch.abs(img) < 1e-6
]
return max(real_eigvalues), min(real_eigvalues)
def test_remote_tensor_multi_var_methods(self):
hook = TorchHook(verbose=False)
local = hook.local_worker
remote = VirtualWorker(hook, 1)
local.add_worker(remote)
x = torch.FloatTensor([[1, 2], [4, 3], [5, 6]])
x.send(remote)
y, z = torch.max(x, 1)
assert torch.equal(y.get(), torch.FloatTensor([2, 4, 6]))
assert torch.equal(z.get(), torch.LongTensor([1, 0, 1]))
x = torch.FloatTensor([[0, 0], [1, 0]]).send(remote)
y, z = torch.qr(x)
assert (y.get() == torch.FloatTensor([[0, -1], [-1, 0]])).all()
assert (z.get() == torch.FloatTensor([[-1, 0], [0, 0]])).all()
x = torch.arange(1, 6).send(remote)
y, z = torch.kthvalue(x, 4)
assert (y.get() == torch.FloatTensor([4])).all()
assert (z.get() == torch.LongTensor([3])).all()
x = torch.FloatTensor([[0, 0], [1, 1]]).send(remote)
y, z = torch.eig(x, True)
assert (y.get() == torch.FloatTensor([[1, 0], [0, 0]])).all()
assert ((z.get() == torch.FloatTensor([[0, 0], [1, 0]])) == torch.ByteTensor([[1, 0], [1, 0]])).all()
x = torch.zeros(3, 3).send(remote)
w, y, z = torch.svd(x)
assert (w.get() == torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])).all()
assert (y.get() == torch.FloatTensor([0, 0, 0])).all()
assert (z.get() == torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])).all()
def reset_parameters(self):
weight_dict = self.state_dict()
for key, value in weight_dict.items():
if key == 'weight_ih_l0':
nn.init.uniform_(value, -1, 1)
value *= self.w_ih_scale[1:]
elif re.fullmatch('weight_ih_l[^0]*', key):
nn.init.uniform_(value, -1, 1)
elif re.fullmatch('bias_ih_l[0-9]*', key):
nn.init.uniform_(value, -1, 1)
value *= self.w_ih_scale[0]
elif re.fullmatch('weight_hh_l[0-9]*', key):
w_hh = torch.Tensor(self.hidden_size * self.hidden_size)
w_hh.uniform_(-1, 1)
if self.density < 1:
zero_weights = torch.randperm(
int(self.hidden_size * self.hidden_size))
zero_weights = zero_weights[:int(self.hidden_size *
self.hidden_size *
(1 - self.density))]
w_hh[zero_weights] = 0
w_hh = w_hh.view(self.hidden_size, self.hidden_size)
abs_eigs = (torch.eig(w_hh)[0]**2).sum(1).sqrt()
weight_dict[key] = w_hh * (self.spectral_radius /
torch.max(abs_eigs))
self.load_state_dict(weight_dict)
def test_remote_tensor_multi_var_methods(self):
hook = TorchHook(verbose=False)
local = hook.local_worker
remote = VirtualWorker(hook, 1)
local.add_worker(remote)
x = torch.FloatTensor([[1, 2], [4, 3], [5, 6]])
x.send(remote)
y, z = torch.max(x, 1)
assert torch.equal(y.get(), torch.FloatTensor([2, 4, 6]))
assert torch.equal(z.get(), torch.LongTensor([1, 0, 1]))
x = torch.FloatTensor([[0, 0], [1, 0]]).send(remote)
y, z = torch.qr(x)
assert (y.get() == torch.FloatTensor([[0, -1], [-1, 0]])).all()
assert (z.get() == torch.FloatTensor([[-1, 0], [0, 0]])).all()
x = torch.arange(1, 6).send(remote)
y, z = torch.kthvalue(x, 4)
assert (y.get() == torch.FloatTensor([4])).all()
assert (z.get() == torch.LongTensor([3])).all()
x = torch.FloatTensor([[0, 0], [1, 1]]).send(remote)
y, z = torch.eig(x, True)
assert (y.get() == torch.FloatTensor([[1, 0], [0, 0]])).all()
assert ((z.get() == torch.FloatTensor(
[[0, 0], [1, 0]], )) == torch.ByteTensor([[1, 0], [1, 0]])).all()
x = torch.zeros(3, 3).send(remote)
w, y, z = torch.svd(x)
assert (w.get() == torch.FloatTensor(
[[1, 0, 0], [0, 1, 0], [0, 0, 1]], )).all()
assert (y.get() == torch.FloatTensor([0, 0, 0])).all()
assert (z.get() == torch.FloatTensor(
[[1, 0, 0], [0, 1, 0], [0, 0, 1]], )).all()
def test_local_tensor_multi_var_methods(self):
x = torch.FloatTensor([[1, 2], [2, 3], [5, 6]])
t, s = torch.max(x, 1)
assert (t == torch.FloatTensor([2, 3, 6])).float().sum() == 3
assert (s == torch.LongTensor([1, 1, 1])).float().sum() == 3
x = torch.FloatTensor([[0, 0], [1, 1]])
y, z = torch.eig(x, True)
assert (y == torch.FloatTensor([[1, 0], [0, 0]])).all()
assert (torch.equal(
z == torch.FloatTensor([[0, 0], [1, 0]], ),
torch.ByteTensor([[1, 0], [1, 0]]),
))
x = torch.FloatTensor([[0, 0], [1, 0]])
y, z = torch.qr(x)
assert (y == torch.FloatTensor([[0, -1], [-1, 0]])).all()
assert (z == torch.FloatTensor([[-1, 0], [0, 0]])).all()
x = torch.arange(1, 6)
y, z = torch.kthvalue(x, 4)
assert (y == torch.FloatTensor([4])).all()
assert (z == torch.LongTensor([3])).all()
x = torch.zeros(3, 3)
w, y, z = torch.svd(x)
assert (w == torch.FloatTensor(
[[1, 0, 0], [0, 1, 0], [0, 0, 1]], )).all()
assert (y == torch.FloatTensor([0, 0, 0])).all()
assert (z == torch.FloatTensor(
[[1, 0, 0], [0, 1, 0], [0, 0, 1]], )).all()
def __force_spectral_radius(self):
# Make the reservoir weight matrix - a unit spectral radius
rad = torch.max(torch.abs(torch.eig(self.W_r)[0]))
self.W_r = self.W_r / rad
# Force spectral radius
self.W_r = self.W_r * self.spectral_radius
def forward(ctx, A):
# normalize the shape to be batched
Ashape = A.shape
if A.ndim == 2:
A = A.unsqueeze(0)
elif A.ndim > 3:
A = A.view(*A.shape[:-2], A.shape[-2], A.shape[-1])
nbatch = A.shape[0]
evecs = torch.empty_like(A).to(A.device)
evals = torch.empty(A.shape[0], A.shape[-1]).to(A.dtype).to(A.device)
for i in range(nbatch):
evalue, evec = torch.eig(A[i], eigenvectors=True)
# check if the eigenvalues contain complex numbers
if not torch.allclose(evalue[:,1], torch.zeros_like(evalue[:,1])):
raise ValueError("The eigenvalues contain complex numbers")
# sort eigenvalues from lowest to highest
evalue, idxsort = torch.sort(evalue[:,0], dim=-1) # idxsort (na,)
evec = torch.index_select(evec, dim=-1, index=idxsort)
evecs[i] = evec
evals[i] = evalue
# reshape the results
evecs = evecs.view(*Ashape)
evals = evals.view(*Ashape[:-1])
ctx.evecs = evecs
ctx.evals = evals
return evals, evecs
def special_sylvester(A, B, d=None):
'''Solves the eqations `AX+XA=B` for positive definite `A`.
This is a special case of Sylvester equation `AX+XB=C`.
https://en.wikipedia.org/wiki/Sylvester_equation
A unique solution exists when `A` and `-A` have no common eigenvalues.
Sources:
https://math.stackexchange.com/a/820313
Explicit solution of the operator equation A*X+X*A=B:
https://core.ac.uk/download/pdf/82315631.pdf
Args:
A: The matrix `A`.
B: The matrix `B`.
d: The eigenvalues or the singular values of `A` if available.
If `d` is provided, `A` must be the eigenvectors, instead.
Returns:
The matrix `X`.
'''
if d is None:
D, Q = torch.eig(A, eigenvectors=True)
d = D[:, 0]
else:
Q = A
C = Q.t().mm(B.mm(Q))
Y = C / (d.view(-1, 1) + d.view(1, -1))
return Q.mm(Y.mm(Q.t()))
def test_torch_function_with_multiple_output_on_remote_var(self):
hook = TorchHook(verbose=False)
me = hook.local_worker
remote = VirtualWorker(id=2, hook=hook)
me.add_worker(remote)
x = Var(torch.FloatTensor([[1, 2], [4, 3], [5, 6]]))
x.send(remote)
y, z = torch.max(x, 1)
y.get()
assert torch.equal(y, Var(torch.FloatTensor([2, 4, 6])))
x = Var(torch.FloatTensor([[0, 0], [1, 0]])).send(remote)
y, z = torch.qr(x)
assert (y.get() == Var(torch.FloatTensor([[0, -1], [-1, 0]]))).all()
assert (z.get() == Var(torch.FloatTensor([[-1, 0], [0, 0]]))).all()
x = Var(torch.arange(1, 6)).send(remote)
y, z = torch.kthvalue(x, 4)
assert (y.get() == Var(torch.FloatTensor([4]))).all()
assert (z.get() == Var(torch.LongTensor([3]))).all()
x = Var(torch.FloatTensor([[0, 0], [0, 0]]))
x.send(remote)
y, z = torch.eig(x, True)
assert (y.get() == Var(torch.FloatTensor([[0, 0], [0, 0]]))).all()
assert (z.get() == Var(torch.FloatTensor([[1, 0.], [0, 1]]))).all()
x = Var(torch.zeros(3, 3)).send(remote)
w, y, z = torch.svd(x)
assert (w.get() == Var(torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))).all()
assert (y.get() == Var(torch.FloatTensor([0, 0, 0]))).all()
assert (z.get() == Var(torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))).all()
def test_torch_function_with_multiple_output_on_local_var(self):
x = Var(torch.FloatTensor([[1, 2], [2, 3], [5, 6]]))
t, s = torch.max(x, 1)
assert (t == Var(torch.FloatTensor([2, 3, 6]))).all()
assert (s == Var(torch.LongTensor([1, 1, 1]))).all()
x = Var(torch.FloatTensor([[0, 0], [0, 0]]))
y, z = torch.eig(x, True)
assert (y == Var(torch.FloatTensor([[0, 0], [0, 0]]))).all()
assert (z == Var(torch.FloatTensor([[1, 0.], [0, 1]]))).all()
x = Var(torch.FloatTensor([[0, 0], [1, 0]]))
y, z = torch.qr(x)
assert (y == Var(torch.FloatTensor([[0, -1], [-1, 0]]))).all()
assert (z == Var(torch.FloatTensor([[-1, 0], [0, 0]]))).all()
x = Var(torch.arange(1, 6))
y, z = torch.kthvalue(x, 4)
assert (y == Var(torch.FloatTensor([4]))).all()
assert (z == Var(torch.LongTensor([3]))).all()
x = Var(torch.zeros(3, 3))
w, y, z = torch.svd(x)
assert (w == Var(torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))).all()
assert (y == Var(torch.FloatTensor([0, 0, 0]))).all()
assert (z == Var(torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))).all()
def test_batch_symeig_top():
# input = torch.randn(8, 4, 4).float().cuda()
num_q = 800
input_lrf = torch.randn(num_q, 8, 4).cuda()
input_lrf = F.normalize(input_lrf, p=2, dim=-1)
input_lrf = input_lrf.view(-1, 4)
test = torch.bmm(input_lrf.unsqueeze(2), input_lrf.unsqueeze(1))
test = test.view(num_q, 8, 4, 4)
input = torch.sum(test, 1)
input3 = input.clone()
input3.requires_grad = True
input.requires_grad = True
# batch wise back
w, v = S.symeig(input)
v_max = v[:, 3].clone()
bool_vmax = v_max[:, 0] / torch.abs(v_max[:, 0])
bool_vmax = bool_vmax.contiguous().view(-1, 1)
bool_vmax = bool_vmax.expand(v_max.size(0), 4)
v_max = v_max * bool_vmax
(v_max.mean()).backward()
# pytorch version loop back
averaged_Q4 = torch.rand(num_q, 4)
for i in range(input3.size(0)):
eigenValues, eigenVectors = torch.eig(input3[i], eigenvectors=True)
e_values, e_indices = torch.max(eigenValues, 0)
averaged_Q4[i] = eigenVectors[:, e_indices]
if (averaged_Q4[i][0] < 0):
averaged_Q4[i] = -averaged_Q4[i]
(averaged_Q4.mean()).backward()
torch.testing.assert_allclose(input.grad, input3.grad)
def get_stably_bounded_shapes(N, n, a, b, c, d, h, w):
'''
This could be made into a class for the data loader.
input : positive integer, non-negative integer, number, number with a <= b, number, number with c <= d, positive integer, positive integer
output : [N, h, w] tensor
'''
x, y = torch.meshgrid(torch.linspace(a, b, h), -torch.linspace(c, d, w))
X = torch.stack([x**(n // 2 - i) * y**i for i in range(n // 2 + 1)])
M = torch.tensor([])
for _ in range(N):
M_i = torch.randn(n // 2 + 1, n // 2 + 1)
M_i = torch.tril(M_i) + torch.tril(M_i, diagonal=-1).t()
while not torch.all(torch.eig(M_i)[0][:, 0] > 0):
M_i = torch.randn(n // 2 + 1, n // 2 + 1)
M_i = torch.tril(M_i) + torch.tril(M_i, diagonal=-1).t()
M = torch.cat((M, M_i.view(1, *M_i.shape)))
image = torch.einsum('ikl,mij,jkl->mkl', X, M, X)
C = torch.randn(N, int(nCk(n + 2, 2)) - n - 1)
B = torch.stack([x**i * y**j for i in range(n) for j in range(n - i)])
return (image + torch.einsum('li,ijk->jk', C, B) < 0).float()
def generate_reservoir(size, radius, degree):
sparsity = degree / float(size)
A = torch.tensor(sparse.rand(size, size, density=sparsity).todense(),
dtype=torch.float32)
e = torch.max(torch.eig(A, eigenvectors=False)[0])
A = (A / e) * radius
return A
def test_torch_function_with_multiple_output_on_local_var(self):
x = Var(torch.FloatTensor([[1, 2], [2, 3], [5, 6]]))
t, s = torch.max(x, 1)
assert (t == Var(torch.FloatTensor([2, 3, 6]))).all()
assert (s == Var(torch.LongTensor([1, 1, 1]))).all()
x = Var(torch.FloatTensor([[0, 0], [0, 0]]))
y, z = torch.eig(x, True)
assert (y == Var(torch.FloatTensor([[0, 0], [0, 0]]))).all()
assert (z == Var(torch.FloatTensor([[1, 0.], [0, 1]]))).all()
x = Var(torch.FloatTensor([[0, 0], [1, 0]]))
y, z = torch.qr(x)
assert (y == Var(torch.FloatTensor([[0, -1], [-1, 0]]))).all()
assert (z == Var(torch.FloatTensor([[-1, 0], [0, 0]]))).all()
x = Var(torch.arange(1, 6))
y, z = torch.kthvalue(x, 4)
assert (y == Var(torch.FloatTensor([4]))).all()
assert (z == Var(torch.LongTensor([3]))).all()
x = Var(torch.zeros(3, 3))
w, y, z = torch.svd(x)
assert (w == Var(torch.FloatTensor(
[[1, 0, 0], [0, 1, 0], [0, 0, 1]], ))).all()
assert (y == Var(torch.FloatTensor([0, 0, 0]))).all()
assert (z == Var(torch.FloatTensor(
[[1, 0, 0], [0, 1, 0], [0, 0, 1]], ))).all()
def test_local_tensor_multi_var_methods(self):
x = torch.FloatTensor([[1, 2], [2, 3], [5, 6]])
t, s = torch.max(x, 1)
assert (t == torch.FloatTensor([2, 3, 6])).float().sum() == 3
assert (s == torch.LongTensor([1, 1, 1])).float().sum() == 3
x = torch.FloatTensor([[0, 0], [1, 1]])
y, z = torch.eig(x, True)
assert (y == torch.FloatTensor([[1, 0], [0, 0]])).all()
assert (torch.equal(z == torch.FloatTensor([[0, 0], [1, 0]]), torch.ByteTensor([[1, 0], [1, 0]])))
x = torch.FloatTensor([[0, 0], [1, 0]])
y, z = torch.qr(x)
assert (y == torch.FloatTensor([[0, -1], [-1, 0]])).all()
assert (z == torch.FloatTensor([[-1, 0], [0, 0]])).all()
x = torch.arange(1, 6)
y, z = torch.kthvalue(x, 4)
assert (y == torch.FloatTensor([4])).all()
assert (z == torch.LongTensor([3])).all()
x = torch.zeros(3, 3)
w, y, z = torch.svd(x)
assert (w == torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])).all()
assert (y == torch.FloatTensor([0, 0, 0])).all()
assert (z == torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])).all()
def priorTrainNN(self, x_train, y_train):
"""
Updates the prior of the Bayesian NN
Args:
X_train (th.DoubleTensor): [N x D_in] column matrix of training inputs
Y_train (th.DoubleTensor): [N x D_out] column matrix of training outputs
"""
x_t = Variable(x_train)
y_t = Variable(y_train, requires_grad=False)
#Get the hessian of the error function and its eigen values
hess = self.getHessian(x_train, y_train)
e, v = th.eig(self.beta * hess.data, eigenvectors=False)
#Compute (w^t)*(w)
d = 0
for param in self.model.parameters():
target = Variable(th.zeros(param.size())).type(dtype)
d += self.reg_fn(param, target).data
#Iterate the alpha calculations to hopefully converge (if hessian is nice to us)
for i in range(20):
gamma = th.sum(e[:, 0] / (self.alpha + e[:, 0]), 0) #Eq. 5.179
self.alpha = (gamma / d)[0] #Eq. 5.178
print(self.alpha)
print('Alpha Updated to: ' + str(self.alpha))
def test_torch_function_with_multiple_output_on_remote_var(self):
hook = TorchHook(verbose=False)
me = hook.local_worker
remote = VirtualWorker(id=2, hook=hook)
me.add_worker(remote)
x = Var(torch.FloatTensor([[1, 2], [4, 3], [5, 6]]))
x.send(remote)
y, z = torch.max(x, 1)
y.get()
assert torch.equal(y, Var(torch.FloatTensor([2, 4, 6])))
x = Var(torch.FloatTensor([[0, 0], [1, 0]])).send(remote)
y, z = torch.qr(x)
assert (y.get() == Var(torch.FloatTensor([[0, -1], [-1, 0]]))).all()
assert (z.get() == Var(torch.FloatTensor([[-1, 0], [0, 0]]))).all()
x = Var(torch.arange(1, 6)).send(remote)
y, z = torch.kthvalue(x, 4)
assert (y.get() == Var(torch.FloatTensor([4]))).all()
assert (z.get() == Var(torch.LongTensor([3]))).all()
x = Var(torch.FloatTensor([[0, 0], [0, 0]]))
x.send(remote)
y, z = torch.eig(x, True)
assert (y.get() == Var(torch.FloatTensor([[0, 0], [0, 0]]))).all()
assert (z.get() == Var(torch.FloatTensor([[1, 0.], [0, 1]]))).all()
x = Var(torch.zeros(3, 3)).send(remote)
w, y, z = torch.svd(x)
assert (w.get() == Var(
torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]], ))).all()
assert (y.get() == Var(torch.FloatTensor([0, 0, 0]))).all()
assert (z.get() == Var(
torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]], ))).all()
def PCA(data, k=2):
X = torch.from_numpy(data)
X_mean = torch.mean(X, 0)
X = X - X_mean.expand_as(X)
v, w = torch.eig(torch.mm(X, torch.t(X)), eigenvectors=True)
return torch.mm(w[:k, :], X)
def get_obs(self, H, SpinOp, solver_params):
# [ESpec, U] = EigenSolver.apply(H)
[ESpec, U] = torch.eig(H, eigenvectors=True)
ESpec = ESpec[:, 0]
Sztot = sum(SpinOp.SzP).double()
Sxtot = sum(SpinOp.SxP).double()
Mnn = torch.diag(U.transpose(1, 0).mm(Sztot).mm(U))
Mnn_x = torch.diag(U.transpose(1, 0).mm(Sxtot).mm(U))
MSqnn = torch.diag(U.transpose(1, 0).mm(Sztot).mm(Sztot).mm(U))
for i in range(self.N):
self.Z[i] = (torch.exp(-self.beta[i] * ESpec)).sum()
for i in range(self.N):
self.Fe[i, 0] = self.beta[i]
self.Fe[i, 1] = -1 / self.beta[i] * torch.log(
(torch.exp(-self.beta[i] * ESpec)).sum())
self.M[i] = (Mnn *
torch.exp(-self.beta[i] * ESpec)).sum() / self.Z[i]
self.M_paral[i] = (
Mnn_x * torch.exp(-self.beta[i] * ESpec)).sum() / self.Z[i]
self.E[i] = (ESpec *
torch.exp(-self.beta[i] * ESpec)).sum() / self.Z[i]
for i in range(self.N):
self.C[i] = ((ESpec ** 2 * torch.exp(-self.beta[i] * ESpec)).sum() / self.Z[i] - self.E[i] ** 2) * \
self.beta[i] ** 2
self.Chi[i] = (
(MSqnn * torch.exp(-self.beta[i] * ESpec)).sum() / self.Z[i] -
self.M[i]**2) * self.beta[i]
self.C = self.C / solver_params['L']
self.Chi = self.Chi / solver_params['L']
self.M = self.M / solver_params['L']
self.M_paral = self.M_paral / solver_params['L']
def PCA_eig(self, X, k, center=True, scale=False):
n, p = X.size()
print(X)
ones = torch.ones(n).view([n, 1])
h = ((1 / n) * torch.mm(ones, ones.t())) if center else torch.zeros(
n * n).view([n, n])
H = torch.eye(n) - h
X_center = torch.mm(H.double(), X.double())
print("X_center", X_center)
covariance = 1 / (n - 1) * torch.mm(X_center.t(), X_center).view(p, p)
print("convariance", covariance)
scaling = torch.sqrt(1 / torch.diag(covariance)).double(
) if scale else torch.ones(p).double()
print("scaling", scaling)
A = torch.diag(scaling).view(p, p)
print("A", A)
print("A size", A.size(), "covariance size", covariance.size())
scaled_covariance = torch.empty(p, p)
scaled_covariance = torch.mm(A, covariance)
print(scaled_covariance)
eigenvalues, eigenvectors = torch.eig(scaled_covariance, True)
components = (eigenvectors[:, :k]).t()
projection = torch.mm(X, components.t())
return projection
def nearest_pd(A, f=np.spacing):
"""Find the nearest positive-definite matrix to input
A Python/Numpy port of John D'Errico's `nearestSPD` MATLAB code [1], which credits [2]
[1] https://www.mathworks.com/matlabcentral/fileexchange/42885-nearestspd
[2] https://doi.org/10.1016/0024-3795(88)90223-6
"""
B = (A + A.T) / 2
_, s, V = torch.svd(B)
# For a comparison with kanga, see the following:
# https://github.com/pytorch/pytorch/issues/16076#issuecomment-477755364
H = torch.matmul(V, torch.matmul(torch.diag(s), V.T))
A2 = (B + H) / 2
A3 = (A2 + A2.T) / 2
if is_pos_def(A3):
return A3
spacing = f(torch.norm(A).item())
I = torch.eye(A.shape[0])
k = 1
while not is_pos_def(A3):
eigenvals = torch.eig(A3, eigenvectors=False)[0][:, 0]
mineig = eigenvals.min().item()
A3 += I * (-mineig * k**2 + spacing)
k += 1
return A3
def lda(x1, x2, device="cpu"):
with torch.no_grad():
x1 = torch.tensor(x1, device=device, dtype=torch.float)
x2 = torch.tensor(x2, device=device, dtype=torch.float)
m1 = torch.mean(x1, dim=0)
m2 = torch.mean(x2, dim=0)
m = (len(x1) * m1 + len(x2) * m2) / (len(x1) + len(x2))
d1 = x1 - m1[None, :]
scatter1 = d1.t() @ d1
d2 = x2 - m2[None, :]
scatter2 = d2.t() @ d2
within_class_scatter = scatter1 + scatter2
d1 = m1 - m[None, :]
scatter1 = len(x1) * (d1.t() @ d1)
d2 = m2 - m[None, :]
scatter2 = len(x2) * (d2.t() @ d2)
between_class_scatter = scatter1 + scatter2
p = torch.pinverse(within_class_scatter) @ between_class_scatter
eigenvalues, eigenvectors = torch.eig(p, eigenvectors=True)
idx = torch.argsort(eigenvalues[:, 0], descending=True)
eigenvalues = eigenvalues[idx, 0]
eigenvectors = eigenvectors[idx, :]
return eigenvectors[0, :].cpu().numpy()
def expm_eig(A):
eigen_values, eigen_vector = th.eig(A, eigenvectors=True)
eigen_values.exp_()
return th.mm(th.mm(eigen_vector, th.diag(eigen_values[:, 0])),
eigen_vector.t_())
def ARCoeffecientGeneration(arCoeffMeans, arCoeffecientNoiseVar, seed=-1):
if (seed > 0):
torch.manual_seed(seed)
arCoeffsMatrix = torch.eye(2)
arCoeffsMatrix[1] = arCoeffsMatrix[0]
goodCoeffs = False
# Pre-Allocating the arCoeffNoise array
arCoeffNoise = torch.empty(2, 1, dtype=torch.float)
eigValsEvaluated = torch.empty(2, dtype=torch.float)
while (not goodCoeffs):
# Generate new AR Coefficients until you get a pair who's eigenvalues are
# less than 1.
# We do this because the system would explode to infinity if the eigenvalues
# were greater than 1.
arCoeffNoise[0] = torch.randn(1) * torch.sqrt(
torch.tensor(arCoeffecientNoiseVar, dtype=torch.float))
arCoeffNoise[1] = torch.randn(1) * torch.sqrt(
torch.tensor(arCoeffecientNoiseVar, dtype=torch.float))
arCoeffsMatrix[0, 0] = arCoeffMeans[0] + arCoeffNoise[0]
arCoeffsMatrix[0, 1] = arCoeffMeans[1] + arCoeffNoise[1]
# Compute EigenValues of F
eigValsEvaluated = torch.abs(
torch.eig(arCoeffsMatrix, eigenvectors=False)[0][0]) > 1
# Determine if any of them have a greater magnitude than 1
if (not eigValsEvaluated.any()):
goodCoeffs = True
return arCoeffsMatrix
def _spectral_embedding(self, X):
"""
Helper function to embed the dataset X into the eigenvectors of the graph Laplacian matrix
Returns
-------
ht.DNDarray, shape=(m_lanczos):
Eigenvalues of the graph's Laplacian matrix.
ht.DNDarray, shape=(n, m_lanczos):
Eigenvectors of the graph's Laplacian matrix.
"""
L = self._laplacian.construct(X)
# 3. Eigenvalue and -vector calculation via Lanczos Algorithm
v0 = ht.full(
(L.shape[0], ),
fill_value=1.0 / math.sqrt(L.shape[0]),
dtype=L.dtype,
split=0,
device=L.device,
)
V, T = ht.lanczos(L, self.n_lanczos, v0)
# 4. Calculate and Sort Eigenvalues and Eigenvectors of tridiagonal matrix T
eval, evec = torch.eig(T._DNDarray__array, eigenvectors=True)
# If x is an Eigenvector of T, then y = V@x is the corresponding Eigenvector of L
eval, idx = torch.sort(eval[:, 0], dim=0)
eigenvalues = ht.array(eval)
eigenvectors = ht.matmul(V, ht.array(evec))[:, idx]
return eigenvalues, eigenvectors
def spectral_radius(m):
"""
Compute spectral radius of a square 2-D tensor
:param m: squared 2D tensor
:return:
"""
return torch.max(torch.abs(torch.eig(m)[0])).item()
def svd_wrapper(matrix, mode, ncomp, debug, verbose, usv=False,
random_state=None, to_numpy=True):
""" Wrapper for different SVD libraries (CPU and GPU).
Parameters
----------
matrix : array_like, 2d
2d input matrix.
mode : {'lapack', 'arpack', 'eigen', 'randsvd', 'cupy', 'eigencupy',
'randcupy', 'pytorch', 'eigenpytorch', 'randpytorch'}, str optional
Switch for the SVD method/library to be used. ``lapack`` uses the LAPACK
linear algebra library through Numpy and it is the most conventional way
of computing the SVD (deterministic result computed on CPU). ``arpack``
uses the ARPACK Fortran libraries accessible through Scipy (computation
on CPU). ``eigen`` computes the singular vectors through the
eigendecomposition of the covariance M.M' (computation on CPU).
``randsvd`` uses the randomized_svd algorithm implemented in Sklearn
(computation on CPU). ``cupy`` uses the Cupy library for GPU computation
of the SVD as in the LAPACK version. ``eigencupy`` offers the same
method as with the ``eigen`` option but on GPU (through Cupy).
``randcupy`` is an adaptation f the randomized_svd algorithm, where all
the computations are done on a GPU (through Cupy). ``pytorch`` uses the
Pytorch library for GPU computation of the SVD. ``eigenpytorch`` offers
the same method as with the ``eigen`` option but on GPU (through
Pytorch). ``randpytorch`` is an adaptation of the randomized_svd
algorithm, where all the linear algebra computations are done on a GPU
(through Pytorch).
ncomp : int
Number of singular vectors to be obtained. In the cases when the full
SVD is computed (LAPACK, ARPACK, EIGEN, CUPY), the matrix of singular
vectors is truncated.
debug : bool
If True the explained variance ratio is computed and displayed.
verbose: bool
If True intermediate information is printed out.
usv : bool optional
If True the 3 terms of the SVD factorization are returned.
random_state : int, RandomState instance or None, optional
If int, random_state is the seed used by the random number generator.
If RandomState instance, random_state is the random number generator.
If None, the random number generator is the RandomState instance used
by np.random. Used for ``randsvd`` mode.
to_numpy : bool, optional
If True (by default) the arrays computed in GPU are transferred from
VRAM and converted to numpy ndarrays.
Returns
-------
V : array_like
The right singular vectors of the input matrix. If ``usv`` is True it
returns the left and right singular vectors and the singular values of
the input matrix.
References
----------
* For ``lapack`` SVD mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.svd.html
http://www.netlib.org/lapack/
* For ``eigen`` mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.eigh.html
* For ``arpack`` SVD mode see:
https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.svds.html
http://www.caam.rice.edu/software/ARPACK/
* For ``randsvd`` SVD mode see:
https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/utils/extmath.py
Finding structure with randomness: Stochastic algorithms for constructing
approximate matrix decompositions
Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061
* For ``cupy`` SVD mode see:
https://docs-cupy.chainer.org/en/stable/reference/generated/cupy.linalg.svd.html
* For ``eigencupy`` mode see:
https://docs-cupy.chainer.org/en/master/reference/generated/cupy.linalg.eigh.html
* For ``pytorch`` SVD mode see:
http://pytorch.org/docs/master/torch.html#torch.svd
* For ``eigenpytorch`` mode see:
http://pytorch.org/docs/master/torch.html#torch.eig
"""
def reconstruction(ncomp, U, S, V, var=1):
if mode == 'lapack':
rec_matrix = np.dot(U[:, :ncomp],
np.dot(np.diag(S[:ncomp]), V[:ncomp]))
rec_matrix = rec_matrix.T
print(' Matrix reconstruction with {} PCs:'.format(ncomp))
print(' Mean Absolute Error =', MAE(matrix, rec_matrix))
print(' Mean Squared Error =', MSE(matrix, rec_matrix))
# see https://github.com/scikit-learn/scikit-learn/blob/c3980bcbabd9d2527548820581725df2904e4a0d/sklearn/decomposition/pca.py
exp_var = (S ** 2) / (S.shape[0] - 1)
full_var = np.sum(exp_var)
explained_variance_ratio = exp_var / full_var # % of variance explained by each PC
ratio_cumsum = np.cumsum(explained_variance_ratio)
elif mode == 'eigen':
exp_var = (S ** 2) / (S.shape[0] - 1)
full_var = np.sum(exp_var)
explained_variance_ratio = exp_var / full_var # % of variance explained by each PC
ratio_cumsum = np.cumsum(explained_variance_ratio)
else:
rec_matrix = np.dot(U, np.dot(np.diag(S), V))
print(' Matrix reconstruction MAE =', MAE(matrix, rec_matrix))
exp_var = (S ** 2) / (S.shape[0] - 1)
full_var = np.var(matrix, axis=0).sum()
explained_variance_ratio = exp_var / full_var # % of variance explained by each PC
if var == 1:
pass
else:
explained_variance_ratio = explained_variance_ratio[::-1]
ratio_cumsum = np.cumsum(explained_variance_ratio)
msg = ' This info makes sense when the matrix is mean centered '
msg += '(temp-mean scaling)'
print(msg)
lw = 2; alpha = 0.4
fig = plt.figure(figsize=vip_figsize)
fig.subplots_adjust(wspace=0.4)
ax1 = plt.subplot2grid((1, 3), (0, 0), colspan=2)
ax1.step(range(explained_variance_ratio.shape[0]),
explained_variance_ratio, alpha=alpha, where='mid',
label='Individual EVR', lw=lw)
ax1.plot(ratio_cumsum, '.-', alpha=alpha,
label='Cumulative EVR', lw=lw)
ax1.legend(loc='best', frameon=False, fontsize='medium')
ax1.set_ylabel('Explained variance ratio (EVR)')
ax1.set_xlabel('Principal components')
ax1.grid(linestyle='solid', alpha=0.2)
ax1.set_xlim(-10, explained_variance_ratio.shape[0] + 10)
ax1.set_ylim(0, 1)
trunc = 20
ax2 = plt.subplot2grid((1, 3), (0, 2), colspan=1)
# plt.setp(ax2.get_yticklabels(), visible=False)
ax2.step(range(trunc), explained_variance_ratio[:trunc], alpha=alpha,
where='mid', lw=lw)
ax2.plot(ratio_cumsum[:trunc], '.-', alpha=alpha, lw=lw)
ax2.set_xlabel('Principal components')
ax2.grid(linestyle='solid', alpha=0.2)
ax2.set_xlim(-2, trunc + 2)
ax2.set_ylim(0, 1)
msg = ' Cumulative explained variance ratio for {} PCs = {:.5f}'
# plt.savefig('figure.pdf', dpi=300, bbox_inches='tight')
print(msg.format(ncomp, ratio_cumsum[ncomp - 1]))
# --------------------------------------------------------------------------
if matrix.ndim != 2:
raise TypeError('Input matrix is not a 2d array')
if usv:
if mode not in ('lapack', 'arpack', 'randsvd', 'cupy', 'randcupy',
'pytorch', 'randpytorch'):
msg = "Returning USV is supported with modes lapack, arpack, "
msg += "randsvd, cupy, randcupy, pytorch or randpytorch"
raise ValueError(msg)
if ncomp > min(matrix.shape[0], matrix.shape[1]):
msg = '{} PCs cannot be obtained from a matrix with size [{},{}].'
msg += ' Increase the size of the patches or request less PCs'
raise RuntimeError(msg.format(ncomp, matrix.shape[0], matrix.shape[1]))
if mode == 'eigen':
# building C as np.dot(matrix.T,matrix) is slower and takes more memory
C = np.dot(matrix, matrix.T) # covariance matrix
e, EV = linalg.eigh(C) # EVals and EVs
pc = np.dot(EV.T, matrix) # PCs using a compact trick when cov is MM'
V = pc[::-1] # reverse since we need the last EVs
S = np.sqrt(np.abs(e)) # SVals = sqrt(EVals)
S = S[::-1] # reverse since EVals go in increasing order
if debug:
reconstruction(ncomp, None, S, None)
for i in range(V.shape[1]):
V[:, i] /= S # scaling EVs by the square root of EVals
V = V[:ncomp]
if verbose:
print('Done PCA with numpy linalg eigh functions')
elif mode == 'lapack':
# n_frames is usually smaller than n_pixels. In this setting taking the SVD of M'
# and keeping the left (transposed) SVs is faster than taking the SVD of M (right SVs)
U, S, V = linalg.svd(matrix.T, full_matrices=False)
if debug:
reconstruction(ncomp, U, S, V)
V = V[:ncomp] # we cut projection matrix according to the # of PCs
U = U[:, :ncomp]
S = S[:ncomp]
if verbose:
print('Done SVD/PCA with numpy SVD (LAPACK)')
elif mode == 'arpack':
U, S, V = svds(matrix, k=ncomp)
if debug:
reconstruction(ncomp, U, S, V, -1)
if verbose:
print('Done SVD/PCA with scipy sparse SVD (ARPACK)')
elif mode == 'randsvd':
U, S, V = randomized_svd(matrix, n_components=ncomp, n_iter=2,
transpose='auto', random_state=random_state)
if debug:
reconstruction(ncomp, U, S, V)
if verbose:
print('Done SVD/PCA with randomized SVD')
elif mode == 'cupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
u_gpu, s_gpu, vh_gpu = cupy.linalg.svd(a_gpu, full_matrices=True,
compute_uv=True)
V = vh_gpu[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if usv:
S = s_gpu[:ncomp]
if to_numpy:
S = cupy.asnumpy(S)
U = u_gpu[:, :ncomp]
if to_numpy:
U = cupy.asnumpy(U)
if verbose:
print('Done SVD/PCA with cupy (GPU)')
elif mode == 'randcupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='cupy')
if to_numpy:
V = cupy.asnumpy(V)
S = cupy.asnumpy(S)
U = cupy.asnumpy(U)
if debug:
reconstruction(ncomp, U, S, V)
if verbose:
print('Done randomized SVD/PCA with cupy (GPU)')
elif mode == 'eigencupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
C = cupy.dot(a_gpu, a_gpu.T) # covariance matrix
e, EV = cupy.linalg.eigh(C) # eigenvalues and eigenvectors
pc = cupy.dot(EV.T, a_gpu) # PCs using a compact trick when cov is MM'
V = pc[::-1] # reverse since last eigenvectors are the ones we want
S = cupy.sqrt(e)[::-1] # reverse since eigenvalues are in increasing order
if debug:
reconstruction(ncomp, None, S, None)
for i in range(V.shape[1]):
V[:, i] /= S # scaling by the square root of eigenvalues
V = V[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if verbose:
print('Done PCA with cupy eigh function (GPU)')
elif mode == 'pytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32').T))
u_gpu, s_gpu, vh_gpu = torch.svd(a_gpu)
V = vh_gpu[:ncomp]
S = s_gpu[:ncomp]
U = torch.transpose(u_gpu, 0, 1)[:ncomp]
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if verbose:
print('Done SVD/PCA with pytorch (GPU)')
elif mode == 'eigenpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32')))
C = torch.mm(a_gpu, torch.transpose(a_gpu, 0, 1))
e, EV = torch.eig(C, eigenvectors=True)
V = torch.mm(torch.transpose(EV, 0, 1), a_gpu)
S = torch.sqrt(e[:, 0])
if debug:
reconstruction(ncomp, None, S, None)
for i in range(V.shape[1]):
V[:, i] /= S
V = V[:ncomp]
if to_numpy:
V = np.array(V)
if verbose:
print('Done PCA with pytorch eig function')
elif mode == 'randpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='pytorch')
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if debug:
reconstruction(ncomp, U, S, V)
if verbose:
print('Done randomized SVD/PCA with randomized pytorch (GPU)')
else:
raise ValueError('The SVD mode is not available')
if usv:
if mode == 'lapack':
return V.T, S, U.T
elif mode == 'pytorch':
if to_numpy:
return V.T, S, U.T
else:
return torch.transpose(V, 0, 1), S, torch.transpose(U, 0, 1)
else:
return U, S, V
else:
if mode == 'lapack':
return U.T
elif mode == 'pytorch':
return U
else:
return V
def svd_wrapper(matrix, mode, ncomp, verbose, full_output=False,
random_state=None, to_numpy=True):
""" Wrapper for different SVD libraries (CPU and GPU).
Parameters
----------
matrix : numpy ndarray, 2d
2d input matrix.
mode : {'lapack', 'arpack', 'eigen', 'randsvd', 'cupy', 'eigencupy',
'randcupy', 'pytorch', 'eigenpytorch', 'randpytorch'}, str optional
Switch for the SVD method/library to be used.
``lapack``: uses the LAPACK linear algebra library through Numpy
and it is the most conventional way of computing the SVD
(deterministic result computed on CPU).
``arpack``: uses the ARPACK Fortran libraries accessible through
Scipy (computation on CPU).
``eigen``: computes the singular vectors through the
eigendecomposition of the covariance M.M' (computation on CPU).
``randsvd``: uses the randomized_svd algorithm implemented in
Sklearn (computation on CPU).
``cupy``: uses the Cupy library for GPU computation of the SVD as in
the LAPACK version. `
`eigencupy``: offers the same method as with the ``eigen`` option
but on GPU (through Cupy).
``randcupy``: is an adaptation of the randomized_svd algorithm,
where all the computations are done on a GPU (through Cupy). `
`pytorch``: uses the Pytorch library for GPU computation of the SVD.
``eigenpytorch``: offers the same method as with the ``eigen``
option but on GPU (through Pytorch).
``randpytorch``: is an adaptation of the randomized_svd algorithm,
where all the linear algebra computations are done on a GPU
(through Pytorch).
ncomp : int
Number of singular vectors to be obtained. In the cases when the full
SVD is computed (LAPACK, ARPACK, EIGEN, CUPY), the matrix of singular
vectors is truncated.
verbose: bool
If True intermediate information is printed out.
full_output : bool optional
If True the 3 terms of the SVD factorization are returned. If ``mode``
is eigen then only S and V are returned.
random_state : int, RandomState instance or None, optional
If int, random_state is the seed used by the random number generator.
If RandomState instance, random_state is the random number generator.
If None, the random number generator is the RandomState instance used
by np.random. Used for ``randsvd`` mode.
to_numpy : bool, optional
If True (by default) the arrays computed in GPU are transferred from
VRAM and converted to numpy ndarrays.
Returns
-------
V : numpy ndarray
The right singular vectors of the input matrix. If ``full_output`` is
True it returns the left and right singular vectors and the singular
values of the input matrix. If ``mode`` is set to eigen then only S and
V are returned.
References
----------
* For ``lapack`` SVD mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.svd.html
http://www.netlib.org/lapack/
* For ``eigen`` mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.eigh.html
* For ``arpack`` SVD mode see:
https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.svds.html
http://www.caam.rice.edu/software/ARPACK/
* For ``randsvd`` SVD mode see:
https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/utils/extmath.py
Finding structure with randomness: Stochastic algorithms for constructing
approximate matrix decompositions
Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061
* For ``cupy`` SVD mode see:
https://docs-cupy.chainer.org/en/stable/reference/generated/cupy.linalg.svd.html
* For ``eigencupy`` mode see:
https://docs-cupy.chainer.org/en/master/reference/generated/cupy.linalg.eigh.html
* For ``pytorch`` SVD mode see:
http://pytorch.org/docs/master/torch.html#torch.svd
* For ``eigenpytorch`` mode see:
http://pytorch.org/docs/master/torch.html#torch.eig
"""
if matrix.ndim != 2:
raise TypeError('Input matrix is not a 2d array')
if ncomp > min(matrix.shape[0], matrix.shape[1]):
msg = '{} PCs cannot be obtained from a matrix with size [{},{}].'
msg += ' Increase the size of the patches or request less PCs'
raise RuntimeError(msg.format(ncomp, matrix.shape[0], matrix.shape[1]))
if mode == 'eigen':
# building C as np.dot(matrix.T,matrix) is slower and takes more memory
C = np.dot(matrix, matrix.T) # covariance matrix
e, EV = linalg.eigh(C) # EVals and EVs
pc = np.dot(EV.T, matrix) # PCs using a compact trick when cov is MM'
V = pc[::-1] # reverse since we need the last EVs
S = np.sqrt(np.abs(e)) # SVals = sqrt(EVals)
S = S[::-1] # reverse since EVals go in increasing order
for i in range(V.shape[1]):
V[:, i] /= S # scaling EVs by the square root of EVals
V = V[:ncomp]
if verbose:
print('Done PCA with numpy linalg eigh functions')
elif mode == 'lapack':
# n_frames is usually smaller than n_pixels. In this setting taking
# the SVD of M' and keeping the left (transposed) SVs is faster than
# taking the SVD of M (right SVs)
U, S, V = linalg.svd(matrix.T, full_matrices=False)
V = V[:ncomp] # we cut projection matrix according to the # of PCs
U = U[:, :ncomp]
S = S[:ncomp]
if verbose:
print('Done SVD/PCA with numpy SVD (LAPACK)')
elif mode == 'arpack':
U, S, V = svds(matrix, k=ncomp)
if verbose:
print('Done SVD/PCA with scipy sparse SVD (ARPACK)')
elif mode == 'randsvd':
U, S, V = randomized_svd(matrix, n_components=ncomp, n_iter=2,
transpose='auto', random_state=random_state)
if verbose:
print('Done SVD/PCA with randomized SVD')
elif mode == 'cupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
u_gpu, s_gpu, vh_gpu = cupy.linalg.svd(a_gpu, full_matrices=True,
compute_uv=True)
V = vh_gpu[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if full_output:
S = s_gpu[:ncomp]
if to_numpy:
S = cupy.asnumpy(S)
U = u_gpu[:, :ncomp]
if to_numpy:
U = cupy.asnumpy(U)
if verbose:
print('Done SVD/PCA with cupy (GPU)')
elif mode == 'randcupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='cupy')
if to_numpy:
V = cupy.asnumpy(V)
S = cupy.asnumpy(S)
U = cupy.asnumpy(U)
if verbose:
print('Done randomized SVD/PCA with cupy (GPU)')
elif mode == 'eigencupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
C = cupy.dot(a_gpu, a_gpu.T) # covariance matrix
e, EV = cupy.linalg.eigh(C) # eigenvalues and eigenvectors
pc = cupy.dot(EV.T, a_gpu) # using a compact trick when cov is MM'
V = pc[::-1] # reverse to get last eigenvectors
S = cupy.sqrt(e)[::-1] # reverse since EVals go in increasing order
for i in range(V.shape[1]):
V[:, i] /= S # scaling by the square root of eigvals
V = V[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if verbose:
print('Done PCA with cupy eigh function (GPU)')
elif mode == 'pytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32').T))
u_gpu, s_gpu, vh_gpu = torch.svd(a_gpu)
V = vh_gpu[:ncomp]
S = s_gpu[:ncomp]
U = torch.transpose(u_gpu, 0, 1)[:ncomp]
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if verbose:
print('Done SVD/PCA with pytorch (GPU)')
elif mode == 'eigenpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32')))
C = torch.mm(a_gpu, torch.transpose(a_gpu, 0, 1))
e, EV = torch.eig(C, eigenvectors=True)
V = torch.mm(torch.transpose(EV, 0, 1), a_gpu)
S = torch.sqrt(e[:, 0])
for i in range(V.shape[1]):
V[:, i] /= S
V = V[:ncomp]
if to_numpy:
V = np.array(V)
if verbose:
print('Done PCA with pytorch eig function')
elif mode == 'randpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='pytorch')
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if verbose:
print('Done randomized SVD/PCA with randomized pytorch (GPU)')
else:
raise ValueError('The SVD `mode` is not recognized')
if full_output:
if mode == 'lapack':
return V.T, S, U.T
elif mode == 'pytorch':
if to_numpy:
return V.T, S, U.T
else:
return torch.transpose(V, 0, 1), S, torch.transpose(U, 0, 1)
elif mode in ('eigen', 'eigencupy', 'eigenpytorch'):
return S, V
else:
return U, S, V
else:
if mode == 'lapack':
return U.T
elif mode == 'pytorch':
return U
else:
return V
