def test_remote_tensor_multi_var_methods(self):
hook = TorchHook(verbose=False)
local = hook.local_worker
remote = VirtualWorker(hook, 1)
local.add_worker(remote)
x = torch.FloatTensor([[1, 2], [4, 3], [5, 6]])
x.send(remote)
y, z = torch.max(x, 1)
assert torch.equal(y.get(), torch.FloatTensor([2, 4, 6]))
assert torch.equal(z.get(), torch.LongTensor([1, 0, 1]))
x = torch.FloatTensor([[0, 0], [1, 0]]).send(remote)
y, z = torch.qr(x)
assert (y.get() == torch.FloatTensor([[0, -1], [-1, 0]])).all()
assert (z.get() == torch.FloatTensor([[-1, 0], [0, 0]])).all()
x = torch.arange(1, 6).send(remote)
y, z = torch.kthvalue(x, 4)
assert (y.get() == torch.FloatTensor([4])).all()
assert (z.get() == torch.LongTensor([3])).all()
x = torch.FloatTensor([[0, 0], [1, 1]]).send(remote)
y, z = torch.eig(x, True)
assert (y.get() == torch.FloatTensor([[1, 0], [0, 0]])).all()
assert ((z.get() == torch.FloatTensor([[0, 0], [1, 0]])) == torch.ByteTensor([[1, 0], [1, 0]])).all()
x = torch.zeros(3, 3).send(remote)
w, y, z = torch.svd(x)
assert (w.get() == torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])).all()
assert (y.get() == torch.FloatTensor([0, 0, 0])).all()
assert (z.get() == torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])).all()
def test_torch_function_with_multiple_output_on_local_var(self):
x = Var(torch.FloatTensor([[1, 2], [2, 3], [5, 6]]))
t, s = torch.max(x, 1)
assert (t == Var(torch.FloatTensor([2, 3, 6]))).all()
assert (s == Var(torch.LongTensor([1, 1, 1]))).all()
x = Var(torch.FloatTensor([[0, 0], [0, 0]]))
y, z = torch.eig(x, True)
assert (y == Var(torch.FloatTensor([[0, 0], [0, 0]]))).all()
assert (z == Var(torch.FloatTensor([[1, 0.], [0, 1]]))).all()
x = Var(torch.FloatTensor([[0, 0], [1, 0]]))
y, z = torch.qr(x)
assert (y == Var(torch.FloatTensor([[0, -1], [-1, 0]]))).all()
assert (z == Var(torch.FloatTensor([[-1, 0], [0, 0]]))).all()
x = Var(torch.arange(1, 6))
y, z = torch.kthvalue(x, 4)
assert (y == Var(torch.FloatTensor([4]))).all()
assert (z == Var(torch.LongTensor([3]))).all()
x = Var(torch.zeros(3, 3))
w, y, z = torch.svd(x)
assert (w == Var(torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))).all()
assert (y == Var(torch.FloatTensor([0, 0, 0]))).all()
assert (z == Var(torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))).all()
def test_local_tensor_multi_var_methods(self):
x = torch.FloatTensor([[1, 2], [2, 3], [5, 6]])
t, s = torch.max(x, 1)
assert (t == torch.FloatTensor([2, 3, 6])).float().sum() == 3
assert (s == torch.LongTensor([1, 1, 1])).float().sum() == 3
x = torch.FloatTensor([[0, 0], [1, 1]])
y, z = torch.eig(x, True)
assert (y == torch.FloatTensor([[1, 0], [0, 0]])).all()
assert (torch.equal(z == torch.FloatTensor([[0, 0], [1, 0]]), torch.ByteTensor([[1, 0], [1, 0]])))
x = torch.FloatTensor([[0, 0], [1, 0]])
y, z = torch.qr(x)
assert (y == torch.FloatTensor([[0, -1], [-1, 0]])).all()
assert (z == torch.FloatTensor([[-1, 0], [0, 0]])).all()
x = torch.arange(1, 6)
y, z = torch.kthvalue(x, 4)
assert (y == torch.FloatTensor([4])).all()
assert (z == torch.LongTensor([3])).all()
x = torch.zeros(3, 3)
w, y, z = torch.svd(x)
assert (w == torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])).all()
assert (y == torch.FloatTensor([0, 0, 0])).all()
assert (z == torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]])).all()
def test_torch_function_with_multiple_output_on_remote_var(self):
hook = TorchHook(verbose=False)
me = hook.local_worker
remote = VirtualWorker(id=2, hook=hook)
me.add_worker(remote)
x = Var(torch.FloatTensor([[1, 2], [4, 3], [5, 6]]))
x.send(remote)
y, z = torch.max(x, 1)
y.get()
assert torch.equal(y, Var(torch.FloatTensor([2, 4, 6])))
x = Var(torch.FloatTensor([[0, 0], [1, 0]])).send(remote)
y, z = torch.qr(x)
assert (y.get() == Var(torch.FloatTensor([[0, -1], [-1, 0]]))).all()
assert (z.get() == Var(torch.FloatTensor([[-1, 0], [0, 0]]))).all()
x = Var(torch.arange(1, 6)).send(remote)
y, z = torch.kthvalue(x, 4)
assert (y.get() == Var(torch.FloatTensor([4]))).all()
assert (z.get() == Var(torch.LongTensor([3]))).all()
x = Var(torch.FloatTensor([[0, 0], [0, 0]]))
x.send(remote)
y, z = torch.eig(x, True)
assert (y.get() == Var(torch.FloatTensor([[0, 0], [0, 0]]))).all()
assert (z.get() == Var(torch.FloatTensor([[1, 0.], [0, 1]]))).all()
x = Var(torch.zeros(3, 3)).send(remote)
w, y, z = torch.svd(x)
assert (w.get() == Var(torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))).all()
assert (y.get() == Var(torch.FloatTensor([0, 0, 0]))).all()
assert (z.get() == Var(torch.FloatTensor([[1, 0, 0], [0, 1, 0], [0, 0, 1]]))).all()
def __wct_core(self, cont_feat, styl_feat):
cFSize = cont_feat.size()
c_mean = torch.mean(cont_feat, 1) # c x (h x w)
c_mean = c_mean.unsqueeze(1).expand_as(cont_feat)
cont_feat = cont_feat - c_mean
iden = torch.eye(cFSize[0]) # .double()
if self.is_cuda:
iden = iden.cuda()
contentConv = torch.mm(cont_feat, cont_feat.t()).div(cFSize[1] - 1) + iden
# del iden
c_u, c_e, c_v = torch.svd(contentConv, some=False)
# c_e2, c_v = torch.eig(contentConv, True)
# c_e = c_e2[:,0]
k_c = cFSize[0]
for i in range(cFSize[0] - 1, -1, -1):
if c_e[i] >= 0.00001:
k_c = i + 1
break
sFSize = styl_feat.size()
s_mean = torch.mean(styl_feat, 1)
styl_feat = styl_feat - s_mean.unsqueeze(1).expand_as(styl_feat)
styleConv = torch.mm(styl_feat, styl_feat.t()).div(sFSize[1] - 1)
s_u, s_e, s_v = torch.svd(styleConv, some=False)
k_s = sFSize[0]
for i in range(sFSize[0] - 1, -1, -1):
if s_e[i] >= 0.00001:
k_s = i + 1
break
c_d = (c_e[0:k_c]).pow(-0.5)
step1 = torch.mm(c_v[:, 0:k_c], torch.diag(c_d))
step2 = torch.mm(step1, (c_v[:, 0:k_c].t()))
whiten_cF = torch.mm(step2, cont_feat)
s_d = (s_e[0:k_s]).pow(0.5)
targetFeature = torch.mm(torch.mm(torch.mm(s_v[:, 0:k_s], torch.diag(s_d)), (s_v[:, 0:k_s].t())), whiten_cF)
targetFeature = targetFeature + s_mean.unsqueeze(1).expand_as(targetFeature)
return targetFeature
def train(self, x):
self.model.train()
o = self.model(x)
loss = torch.mean(torch.pow(torch.mm(o, self.B.t()) - x, 2))
self.optimizer.zero_grad()
loss.backward()
self.optimizer.step()
U, _, V = torch.svd(torch.mm(x.t().data, o.data))
self.B = torch.autograd.Variable(torch.mm(U, V.t()))
return loss.data.cpu().numpy()
def func(): torch.svd(torch.rand((N, N)).cuda()) torch.cuda.synchronize()
def find_homography_dlt(
points1: torch.Tensor,
points2: torch.Tensor,
weights: Optional[torch.Tensor] = None) -> torch.Tensor:
r"""Compute the homography matrix using the DLT formulation.
The linear system is solved by using the Weighted Least Squares Solution for the 4 Points algorithm.
Args:
points1: A set of points in the first image with a tensor shape :math:`(B, N, 2)`.
points2: A set of points in the second image with a tensor shape :math:`(B, N, 2)`.
weights: Tensor containing the weights per point correspondence with a shape of :math:`(B, N)`.
Returns:
the computed homography matrix with shape :math:`(B, 3, 3)`.
"""
if points1.shape != points2.shape:
raise AssertionError(points1.shape)
if not (len(points1.shape) >= 1 and points1.shape[-1] == 2):
raise AssertionError(points1.shape)
if points1.shape[1] < 4:
raise AssertionError(points1.shape)
device, dtype = _extract_device_dtype([points1, points2])
eps: float = 1e-8
points1_norm, transform1 = normalize_points(points1)
points2_norm, transform2 = normalize_points(points2)
x1, y1 = torch.chunk(points1_norm, dim=-1, chunks=2) # BxNx1
x2, y2 = torch.chunk(points2_norm, dim=-1, chunks=2) # BxNx1
ones, zeros = torch.ones_like(x1), torch.zeros_like(x1)
# DIAPO 11: https://www.uio.no/studier/emner/matnat/its/nedlagte-emner/UNIK4690/v16/forelesninger/lecture_4_3-estimating-homographies-from-feature-correspondences.pdf # noqa: E501
ax = torch.cat(
[zeros, zeros, zeros, -x1, -y1, -ones, y2 * x1, y2 * y1, y2], dim=-1)
ay = torch.cat(
[x1, y1, ones, zeros, zeros, zeros, -x2 * x1, -x2 * y1, -x2], dim=-1)
A = torch.cat((ax, ay), dim=-1).reshape(ax.shape[0], -1, ax.shape[-1])
if weights is None:
# All points are equally important
A = A.transpose(-2, -1) @ A
else:
# We should use provided weights
if not (len(weights.shape) == 2
and weights.shape == points1.shape[:2]):
raise AssertionError(weights.shape)
w_diag = torch.diag_embed(
weights.unsqueeze(dim=-1).repeat(1, 1,
2).reshape(weights.shape[0], -1))
A = A.transpose(-2, -1) @ w_diag @ A
try:
_, _, V = torch.svd(A)
except RuntimeError:
warnings.warn('SVD did not converge', RuntimeWarning)
return torch.empty((points1_norm.size(0), 3, 3),
device=device,
dtype=dtype)
H = V[..., -1].view(-1, 3, 3)
H = transform2.inverse() @ (H @ transform1)
H_norm = H / (H[..., -1:, -1:] + eps)
return H_norm
def get_loss_components(self, data_dict):
loss_components = dict()
N_data = data_dict['data'].shape[0]
if ('siam_reflection_data' in data_dict) and ('siam_same_levelvec_data'
in data_dict):
is_computing_signed_siamese_pairs = True
else:
is_computing_signed_siamese_pairs = False
data_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['data']).requires_grad_().to(self.device)
norm_level_data_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['norm_level_data']).requires_grad_().to(self.device)
norm_level_weight_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['norm_level_weight']).requires_grad_().to(self.device)
[y_torch, J_torch] = self.y_torch_and_J_torch(data_torch)
# (level set) prediction error
norm_level_wnmse_per_dim = utils.compute_wnmse_per_dim(
prediction=torch.norm(y_torch, dim=1).unsqueeze(1),
ground_truth=norm_level_data_torch,
weight=norm_level_weight_torch)
if is_computing_signed_siamese_pairs:
siam_reflection_data_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['siam_reflection_data']).requires_grad_().to(
self.device)
siam_reflection_weight_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['siam_reflection_weight']).requires_grad_().to(
self.device)
siam_same_levelvec_data_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['siam_same_levelvec_data']).requires_grad_().to(
self.device)
siam_same_levelvec_weight_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['siam_same_levelvec_weight']).requires_grad_().to(
self.device)
augmenting_vector_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['augmenting_vector']).requires_grad_().to(
self.device)
siam_frac_aug_weight_torch = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['siam_frac_aug_weight']).requires_grad_().to(
self.device)
siam_reflection_y_torch = self.y_torch(siam_reflection_data_torch)
siam_reflection_wnmse_torch = utils.compute_wnmse_per_dim(
prediction=(-siam_reflection_y_torch),
ground_truth=y_torch,
weight=siam_reflection_weight_torch)
N_siam_same_levelvec = siam_same_levelvec_data_torch.shape[1]
siam_same_levelvec_y_torch_list = [
self.y_torch(
siam_same_levelvec_data_torch[:, n_siam_same_levelvec, :])
for n_siam_same_levelvec in range(N_siam_same_levelvec)
]
siam_same_levelvec_wnmse_torch = torch.stack([
utils.compute_wnmse_per_dim(
prediction=siam_same_levelvec_y_torch_list[
n_siam_same_levelvec],
ground_truth=y_torch,
weight=siam_same_levelvec_weight_torch[:,
n_siam_same_levelvec]
).squeeze()
for n_siam_same_levelvec in range(N_siam_same_levelvec)
],
dim=0).mean(axis=0)
N_siam_frac_augs = 4
normalized_y_torch = utils.normalize(y_torch, data_axis=1)
siam_frac_aug_normalized_y_torch_list = [
utils.normalize(self.y_torch(data_torch - (
((1.0 * n_siam_frac_augs) / N_siam_frac_augs) *
augmenting_vector_torch)),
data_axis=1)
for n_siam_frac_augs in range(1, N_siam_frac_augs)
]
siam_frac_aug_wnmse_torch = torch.stack([
utils.compute_wnmse_per_dim(
prediction=siam_frac_aug_normalized_y_torch_list[
n_siam_frac_augs],
ground_truth=normalized_y_torch,
weight=siam_frac_aug_weight_torch).squeeze()
for n_siam_frac_augs in range(N_siam_frac_augs - 1)
],
dim=0).mean(axis=0)
# Local PCA's Rowspace and Nullspace Eigenvectors extraction
# (originally from the V Matrix of the Local PCA's Covariance Matrix's SVD (cov_svd_V)):
cov_torch_rowspace = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['cov_rowspace']).to(self.device)
assert (cov_torch_rowspace.shape[2] == (self.dim_ambient -
self.N_constraints))
cov_torch_nullspace = utils.convert_into_at_least_2d_pytorch_tensor(
data_dict['cov_nullspace']).to(self.device)
assert (cov_torch_nullspace.shape[2] == self.N_constraints)
cov_torch_rowspace_projector = (
cov_torch_rowspace @ cov_torch_rowspace.transpose(-2, -1))
cov_torch_nullspace_projector = (
cov_torch_nullspace @ cov_torch_nullspace.transpose(-2, -1))
# Constraint Manifold Neural Network's Jacobian
# Rowspace and Nullspace eigenvectors extraction:
[_, _, J_torch_svd_V] = torch.svd(J_torch, some=False)
J_torch_rowspace = J_torch_svd_V[:, :, :self.N_constraints]
J_torch_nullspace = J_torch_svd_V[:, :, self.N_constraints:]
J_torch_rowspace_projector = (
J_torch_rowspace @ J_torch_rowspace.transpose(-2, -1))
J_torch_nullspace_projector = (
J_torch_nullspace @ J_torch_nullspace.transpose(-2, -1))
# we want to align so that J_torch_nullspace == cov_torch_rowspace,
# so here is the projection loss (I - A^{+}A)b_i whose norm is to be minimized during training:
J_nspace_proj_error_per_dim = cov_torch_nullspace_projector @ J_torch_nullspace
# we want to align so that cov_torch_nullspace == J_torch_rowspace,
# so here is the projection loss (I - B^{+}B)a_j whose norm is to be minimized during training:
cov_nspace_proj_error_per_dim = J_torch_nullspace_projector @ cov_torch_nullspace
# similarly we want to align so that J_torch_rowspace == cov_torch_nullspace:
J_rspace_proj_error_per_dim = cov_torch_rowspace_projector @ J_torch_rowspace
# similarly we want to align so that cov_torch_rowspace == J_torch_nullspace:
cov_rspace_proj_error_per_dim = J_torch_rowspace_projector @ cov_torch_rowspace
# Local Tangent Space Alignment (LTSA) and Local PCA (eigenvectors) alignment errors:
J_nspace_proj_loss_per_dim = (J_nspace_proj_error_per_dim**2).mean(
axis=0)
cov_nspace_proj_loss_per_dim = (cov_nspace_proj_error_per_dim**2).mean(
axis=0)
J_rspace_proj_loss_per_dim = (J_rspace_proj_error_per_dim**2).mean(
axis=0)
cov_rspace_proj_loss_per_dim = (cov_rspace_proj_error_per_dim**2).mean(
axis=0)
loss_components['norm_level_wnmse_per_dim'] = norm_level_wnmse_per_dim
if is_computing_signed_siamese_pairs:
loss_components[
'siam_reflection_wnmse_torch'] = siam_reflection_wnmse_torch
loss_components[
'siam_same_levelvec_wnmse_torch'] = siam_same_levelvec_wnmse_torch
loss_components[
'siam_frac_aug_wnmse_torch'] = siam_frac_aug_wnmse_torch
loss_components[
'J_nspace_proj_loss_per_dim'] = J_nspace_proj_loss_per_dim
loss_components[
'cov_nspace_proj_loss_per_dim'] = cov_nspace_proj_loss_per_dim
loss_components[
'J_rspace_proj_loss_per_dim'] = J_rspace_proj_loss_per_dim
loss_components[
'cov_rspace_proj_loss_per_dim'] = cov_rspace_proj_loss_per_dim
return loss_components
# plt.imshow(ims, interpolation=None)
#
#plt.savefig('mouse_results/clusterings.png', format='png')
#plt.savefig(shared_path + 'clusterings.eps', format='eps')
#mean_hhat = all_hhats_cat.mean(0, keepdim=True)
#all_hhats_cat_c = all_hhats_cat - mean_hhat
#
#U, S, V = torch.svd(all_hhats_cat_c.t())
folder = 'registered_results'
if 1:
mean_x = all_xs_cat.mean(0, keepdim=True)
all_xs_cat_c = (all_xs_cat - mean_x).view(all_xs_cat.size(0), -1)
Ureal, Sreal, _ = torch.svd(all_xs_cat_c.t())
mdl.train(mode=False)
mdl.eval()
W = Ureal[:, :100]
pca_coefs = torch.matmul(W.t(), all_xs_cat_c.t()).t()
recons = torch.matmul(W, pca_coefs.t()).t().contiguous().view(
-1, 456, 320) + mean_x
all_mmds = []
all_stats = []
num_samples = 5
for J in range(1, 60, 5):
print(J)
def randomized_svd_gpu(M, n_components, n_oversamples=10, n_iter='auto',
transpose='auto', random_state=0, lib='cupy'):
"""Computes a truncated randomized SVD on GPU. Adapted from Sklearn.
Parameters
----------
M : ndarray or sparse matrix
Matrix to decompose
n_components : int
Number of singular values and vectors to extract.
n_oversamples : int (default is 10)
Additional number of random vectors to sample the range of M so as
to ensure proper conditioning. The total number of random vectors
used to find the range of M is n_components + n_oversamples. Smaller
number can improve speed but can negatively impact the quality of
approximation of singular vectors and singular values.
n_iter : int or 'auto' (default is 'auto')
Number of power iterations. It can be used to deal with very noisy
problems. When 'auto', it is set to 4, unless `n_components` is small
(< .1 * min(X.shape)) `n_iter` in which case is set to 7.
This improves precision with few components.
transpose : True, False or 'auto' (default)
Whether the algorithm should be applied to M.T instead of M. The
result should approximately be the same. The 'auto' mode will
trigger the transposition if M.shape[1] > M.shape[0] since this
implementation of randomized SVD tend to be a little faster in that
case.
random_state : int, RandomState instance or None, optional (default=None)
The seed of the pseudo random number generator to use when shuffling
the data. If int, random_state is the seed used by the random number
generator; If RandomState instance, random_state is the random number
generator; If None, the random number generator is the RandomState
instance used by `np.random`.
lib : {'cupy', 'pytorch'}, str optional
Chooses the GPU library to be used.
Notes
-----
This algorithm finds a (usually very good) approximate truncated
singular value decomposition using randomization to speed up the
computations. It is particularly fast on large matrices on which
you wish to extract only a small number of components. In order to
obtain further speed up, `n_iter` can be set <=2 (at the cost of
loss of precision).
References
----------
* Finding structure with randomness: Stochastic algorithms for constructing
approximate matrix decompositions
Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061
* A randomized algorithm for the decomposition of matrices
Per-Gunnar Martinsson, Vladimir Rokhlin and Mark Tygert
* An implementation of a randomized algorithm for principal component
analysis
A. Szlam et al. 2014
"""
random_state = check_random_state(random_state)
n_random = n_components + n_oversamples
n_samples, n_features = M.shape
if n_iter == 'auto':
# Checks if the number of iterations is explicitly specified
n_iter = 7 if n_components < .1 * min(M.shape) else 4
if transpose == 'auto':
transpose = n_samples < n_features
if transpose:
M = M.T # this implementation is a bit faster with smaller shape[1]
if lib == 'cupy':
M = cupy.array(M)
M = cupy.asarray(M)
# Generating normal random vectors with shape: (M.shape[1], n_random)
Q = random_state.normal(size=(M.shape[1], n_random))
Q = cupy.array(Q)
Q = cupy.asarray(Q)
# Perform power iterations with Q to further 'imprint' the top
# singular vectors of M in Q
for i in range(n_iter):
Q = cupy.dot(M, Q)
Q = cupy.dot(M.T, Q)
# Sample the range of M by linear projection of Q.
# Extract an orthonormal basis
Q, _ = cupy.linalg.qr(cupy.dot(M, Q), mode='reduced')
# project M to the (k + p) dimensional space using the basis vectors
B = cupy.dot(Q.T, M)
B = cupy.array(B)
Q = cupy.array(Q)
# compute the SVD on the thin matrix: (k + p) wide
Uhat, s, V = cupy.linalg.svd(B, full_matrices=False, compute_uv=True)
del B
U = cupy.dot(Q, Uhat)
if transpose:
# transpose back the results according to the input convention
return (V[:n_components, :].T, s[:n_components],
U[:,:n_components].T)
else:
return U[:, :n_components], s[:n_components], V[:n_components, :]
elif lib == 'pytorch':
M_gpu = torch.Tensor.cuda(torch.from_numpy(M.astype('float32')))
# Generating normal random vectors with shape: (M.shape[1], n_random)
Q = torch.cuda.FloatTensor(M_gpu.shape[1], n_random).normal_()
# Perform power iterations with Q to further 'imprint' the top
# singular vectors of M in Q
for i in range(n_iter):
Q = torch.mm(M_gpu, Q)
Q = torch.mm(torch.transpose(M_gpu, 0, 1), Q)
# Sample the range of M by linear projection of Q.
# Extract an orthonormal basis
Q, _ = torch.qr(torch.mm(M_gpu, Q))
# project M to the (k + p) dimensional space using the basis vectors
B = torch.mm(torch.transpose(Q, 0, 1), M_gpu)
# compute the SVD on the thin matrix: (k + p) wide
Uhat, s, V = torch.svd(B)
del B
U = torch.mm(Q, Uhat)
if transpose:
# transpose back the results according to the input convention
return (torch.transpose(V[:n_components, :], 0, 1),
s[:n_components],
torch.transpose(U[:, :n_components], 0, 1))
else:
return U[:, :n_components], s[:n_components], V[:n_components, :]
def read_matdataset(self):
path = self.datadir + 'feature_map_ResNet_101_{}.hdf5'.format(
self.dataset)
print('_____')
print(path)
tic = time.clock()
hf = h5py.File(path, 'r')
features = np.array(hf.get('feature_map'))
# shape = features.shape
# features = features.reshape(shape[0],shape[1],shape[2]*shape[3])
labels = np.array(hf.get('labels'))
trainval_loc = np.array(hf.get('trainval_loc'))
# train_loc = np.array(hf.get('train_loc')) #--> train_feature = TRAIN SEEN
# val_unseen_loc = np.array(hf.get('val_unseen_loc')) #--> test_unseen_feature = TEST UNSEEN
test_seen_loc = np.array(hf.get('test_seen_loc'))
test_unseen_loc = np.array(hf.get('test_unseen_loc'))
if self.is_unsupervised_attr:
print('Unsupervised Attr')
class_path = './w2v/{}_class.pkl'.format(self.dataset)
with open(class_path, 'rb') as f:
w2v_class = pickle.load(f)
assert w2v_class.shape == (50, 300)
w2v_class = torch.tensor(w2v_class).float()
U, s, V = torch.svd(w2v_class)
reconstruct = torch.mm(torch.mm(U, torch.diag(s)),
torch.transpose(V, 1, 0))
print('sanity check: {}'.format(
torch.norm(reconstruct - w2v_class).item()))
print('shape U:{} V:{}'.format(U.size(), V.size()))
print('s: {}'.format(s))
self.w2v_att = torch.transpose(V, 1, 0).to(self.device)
self.att = torch.mm(U, torch.diag(s)).to(self.device)
self.normalize_att = torch.mm(U, torch.diag(s)).to(self.device)
else:
print('Expert Attr')
att = np.array(hf.get('att'))
print("threshold at zero attribute with negative value")
att[att < 0] = 0
self.att = torch.from_numpy(att).float().to(self.device)
original_att = np.array(hf.get('original_att'))
self.original_att = torch.from_numpy(original_att).float().to(
self.device)
w2v_att = np.array(hf.get('w2v_att'))
self.w2v_att = torch.from_numpy(w2v_att).float().to(self.device)
self.normalize_att = self.original_att / 100
print('Finish loading data in ', time.clock() - tic)
train_feature = features[trainval_loc]
test_seen_feature = features[test_seen_loc]
test_unseen_feature = features[test_unseen_loc]
if self.is_scale:
scaler = preprocessing.MinMaxScaler()
train_feature = scaler.fit_transform(train_feature)
test_seen_feature = scaler.fit_transform(test_seen_feature)
test_unseen_feature = scaler.fit_transform(test_unseen_feature)
train_feature = torch.from_numpy(
train_feature).float() #.to(self.device)
test_seen_feature = torch.from_numpy(
test_seen_feature) #.float().to(self.device)
test_unseen_feature = torch.from_numpy(
test_unseen_feature) #.float().to(self.device)
train_label = torch.from_numpy(
labels[trainval_loc]).long() #.to(self.device)
test_unseen_label = torch.from_numpy(
labels[test_unseen_loc]) #.long().to(self.device)
test_seen_label = torch.from_numpy(
labels[test_seen_loc]) #.long().to(self.device)
self.seenclasses = torch.from_numpy(
np.unique(train_label.cpu().numpy())).to(self.device)
self.unseenclasses = torch.from_numpy(
np.unique(test_unseen_label.cpu().numpy())).to(self.device)
self.ntrain = train_feature.size()[0]
self.ntrain_class = self.seenclasses.size(0)
self.ntest_class = self.unseenclasses.size(0)
self.train_class = self.seenclasses.clone()
self.allclasses = torch.arange(0, self.ntrain_class +
self.ntest_class).long()
# self.train_mapped_label = map_label(train_label, self.seenclasses)
self.data = {}
self.data['train_seen'] = {}
self.data['train_seen']['resnet_features'] = train_feature
self.data['train_seen']['labels'] = train_label
self.data['train_unseen'] = {}
self.data['train_unseen']['resnet_features'] = None
self.data['train_unseen']['labels'] = None
self.data['test_seen'] = {}
self.data['test_seen']['resnet_features'] = test_seen_feature
self.data['test_seen']['labels'] = test_seen_label
self.data['test_unseen'] = {}
self.data['test_unseen']['resnet_features'] = test_unseen_feature
self.data['test_unseen']['labels'] = test_unseen_label
def get_clean_dataloader(self):
"""
Get the feature representation of samples from each class individually.
Compute centered representation.
Singular value decomposition.
Compute the vector of the outlier scores.
Remove the examples with the top epsilon scores from the samples of each class and form the clean dataset.
Returns:
torch.utils.data.DataLoader: after removing the suspicious samples with bigger singular value, return the clean dataloader.
"""
final_set = None # TODO
for k in range(self.dataset.num_classes):
# self.class_dataset = self.dataset.get_class_set(self.mix_dataset,classes = [k])
idx = []
for i, data in enumerate(self.mix_dataset):
_input, _label = self.model.get_data(data)
_input = _input.view(1, _input.shape[0], _input.shape[1],
_input.shape[2])
if _label.item() == k:
idx.append(k)
self.class_dataset = torch.utils.data.Subset(self.mix_dataset, idx)
layer_output_all = torch.empty([]) # TODO
for i, data in enumerate(self.class_dataset):
_input, _label = self.model.get_data(data)
layer_output = self.model.get_layer(
_input, layer_output=self.preprocess_layer)
layer_output = layer_output.view(1, -1)
if i == 0:
layer_output_all = layer_output
else:
layer_output_all = torch.cat(
(layer_output_all, layer_output))
layer_output_mean = torch.mean(layer_output_all, dim=0)
for i in range(len(self.class_dataset)):
layer_output_all[i] = layer_output_all[i] - layer_output_mean
u, s, v = torch.svd(layer_output_all)
v_transpose = torch.transpose(v, 1, 0)
outlier_scores = torch.rand([layer_output_all.shape[0]],
device=env['device'])
for i in range(len(self.class_dataset)):
outlier_scores[i] = (layer_output_all[i].view(1, -1)
@ v_transpose[i].view(-1, 1))**2
outlier_scores_sorted, indices = torch.sort(outlier_scores,
descending=True)
clean_indices = indices[self.epsilon:]
self.class_dataset = torch.utils.data.Subset(
self.class_dataset, clean_indices)
if k == 0:
final_set = self.class_dataset
else:
final_set = torch.utils.data.ConcatDataset(
[final_set, self.class_dataset])
final_dataloader = self.dataset.get_dataloader(mode=None,
dataset=final_set,
num_workers=0,
pin_memory=False)
return final_dataloader
def RealDataAlg(MaxIters,
X,
Y,
R,
sXs,
conDenfs,
Cb=10,
CT=1,
log=0,
bThetainit=None,
betainit=None,
tols=None,
ErrOpts=0,
etab=0.05,
etaT=0.05):
"""
MaxIters: max iteration number.
X: the covariate matrix, n x m x p
Y: the response matrix, n x m
R: the Missing matrix, n x m
sXs: sample of X for MCMC, p x N
conDenfs: a list to contain the likelihood function of Y|X, and its fisrt derivative and second derivative w.r.t second argument.
[f, f2, f22]. In fact, f22 is not used.
Cb: the constant of Lambda_beta
CT: the constant of Lambda_bTheta
log: Whether output detail training log. 0 not output, 1 output simple training log, 2 output detailed training log.
bThetainit: initial value of bTheta
tol: terminate tolerace.
ErrOpts: whether output errors of beta and bTheta. 0 no, 1 yes
etabinit: The initial learning rate of beta
etaTinit: The initial learning rate of btheta
"""
n, m, p = X.shape
f, f2, f22 = conDenfs
tol, tolb, tolT = tols
Lcon = -10
# To contain the training errors of bTheta and beta, respectively.
Likelis = []
bThetahats = []
betahats = []
# Initial the value of beta, bTheta and R_b
bThetaOld = torch.rand(n, m) if bThetainit is None else bThetainit
betaOld = torch.rand(p) if betainit is None else betainit
# the relative change of Loss, i.e. |L_k - L_k+1|/max(|L_k|, |L_k+1|, 1), here the L_k and L_k+1 are with penalty items.
reCh = 1
# Under Cb and CT, compute the Lambda_beta and Lambda_bTheta
LamT = LamTfn(CT, n, m, p)
Lamb = Lambfn(Cb, n, m)
# The log output, nothing to do with algorithm.
if log >= 0:
tb1 = PrettyTable(["Basic Value", "Lamb", "LamT"])
tb1.add_row(["", f"{Lamb.item():>5.3g}", f"{LamT.item():>5.3g}"])
print(tb1)
# The loss, i.e. L + Lamdab_bTheta * ||bTheta||
Losses = []
# Starting optimizing.
for t in tqdm(range(MaxIters), desc="MNAR"):
#--------------------------------------------------------------------------------
# To get the number of nonzeros entry in betaOld
NumN0Old = p - (betaOld.abs() == 0).sum().to(dtorchdtype)
#--------------------------------------------------------------------------------
# compute the loss function (with penalty items) under betaOld and bThetaOld
# Compute L (without penalty items)
LvNow = missdepL(bThetaOld, betaOld, f, X, Y, R, sXs)
# Add L with penalty items.
LossNow = missdepLR(LvNow, bThetaOld, betaOld, LamT, Lamb)
QOld = (LossNow - Lcon) / Lamb
Losses.append(LossNow.item())
#--------------------------------------------------------------------------------
# This block is to update beta.
LpbvOld = missdepLpb(bThetaOld, betaOld, conDenfs, X, Y, R, sXs)
# compute the learning rate of beta
etabOld = etab # 0.05 for linear setting
betaNewRaw = betaOld - etabOld * LpbvOld
# Using rho function to soften updated beta
betaNew = SoftTO(betaNewRaw, etabOld * Lamb)
#--------------------------------------------------------------------------------
# To get the number of nonzeros entry in betaNew
NumN0New = p - (betaNew.abs() == 0).sum().to(dtorchdtype)
#--------------------------------------------------------------------------------
# Update bTheta
LpTvOld = missdepLpT(bThetaOld, betaNew, conDenfs, X, Y, R, sXs)
LvNew = missdepL(bThetaOld, betaNew, f, X, Y, R, sXs)
LossNew = missdepLR(LvNew, bThetaOld, betaNew, LamT, Lamb)
ROld = (LossNew - Lcon) / LamT
etaTOld = etaT # 0.05 for linear setting
#tmpmatrix = bThetaOld-LpTvOld*etaTOld
#tmpmatarr = tmpmatrix.cpu().numpy()
#U, S, VT = np.linalg.svd(tmpmatarr)
#U, S, VT = torch.tensor(U), torch.tensor(S), torch.tensor(VT)
#V = VT.t()
svdres = torch.svd(bThetaOld - LpTvOld * etaTOld)
U, S, V = svdres.U, svdres.S, svdres.V
softS = (S - LamT * etaTOld).clamp_min(0)
bThetaNew = U.matmul(torch.diag(softS)).matmul(V.t())
#--------------------------------------------------------------------------------
# compute the relative change of Loss
if t >= 1:
Lk1 = Losses[-1]
Lk = Losses[-2]
reCh = np.abs(Lk1 - Lk) / np.max(np.abs((Lk, Lk1, 1)))
#--------------------------------------------------------------------------------
# This block is for log output and Error save, nothing to do with the algorithm
if ErrOpts:
Likelis.append(LvNow.item())
bThetahats.append(bThetaOld.norm().item())
betahats.append(betaOld.norm().item())
if log == 1:
tb2 = PrettyTable(["Iteration", "etaT", "Loss"])
tb2.add_row([
f"{t+1:>6}/{MaxIters}", f"{etaTOld:>8.3g}",
f"{Losses[-1]:>8.3f}"
])
print(tb2)
if log == 2:
tb2 = PrettyTable([
"Iter", "etaT", "etab", "Loss", "-lkd", "reCh", "betat Norm",
"Thetat Norm", "beta diff Norm", "btheta diff Norm",
"betat L0 norm"
])
tb2.add_row([
f"{t+1:>4}/{MaxIters}", f"{etaTOld:>3.3g}", f"{etabOld:>3.3g}",
f"{Losses[-1]:>6.3f}", f"{LvNow.item():>2.1g}",
f"{reCh:>6.4g}", f"{betaNew.norm().item():>2.1f}",
f"{bThetaNew.norm().item():>2.1f}",
f"{(betaOld-betaNew).norm().item():>6.3g}",
f"{(bThetaOld-bThetaNew).norm().item():>6.3g}",
f"{NumN0New.item()}"
])
print(tb2)
#--------------------------------------------------------------------------------
# if reCh is smaller than tolerance, stop the loop
if t >= 1:
if (reCh < tol):
break
# if the difference of 2 consecutive bThetahat is smaller than tolerance, stop the loop
if ((bThetaOld - bThetaNew).norm() < tolT) and (
(betaOld - betaNew).norm() < tolb):
break
#--------------------------------------------------------------------------------
# Change New to Old for starting next iteration
#print(betaOld)
#print(softS)
betaOld, bThetaOld = betaNew, bThetaNew
#--------------------------------------------------------------------------------
if ErrOpts:
return betaOld, bThetaOld, t + 1, betahats, bThetahats, Likelis
else:
return betaOld, bThetaOld, t + 1
def set_weight(self, A, bias):
U, S, V = torch.svd(A)
self.B = torch.mm(U, torch.diag(S))
self.V = V.t().contiguous()
self.bias = bias.view(-1, 1)
def train(self,
dataloader,
num_train_updates,
val_dataloader=None,
verbose=False):
"""
Trains the dynamics model on data.
Inputs: dataloader: torch DataLoader that returns batches of samples that can
be indexed by 'x', 'u', and 'xp'
num_train_updates: int specifying the number of gradient steps to take
val_dataloader: torch DataLoader, a dataloader used for validation (optional)
verbose: bool, whether to print training progress.
Progress is logged to a tensorboard summary writer.
Outputs: None. self.model is modified.
"""
self.reset()
config = self.model.config
validation_freq = 100
val_iters = 5
print_freq = validation_freq if verbose else num_train_updates + 1
eval_method = self.evaluate_sample_singlestep
if config['multistep_training']:
eval_method = self.evaluate_sample_multistep
data_iter = iter(dataloader)
with trange(num_train_updates,
disable=(not verbose or not config['tqdm'])) as pbar:
for idx in pbar:
try:
sample = next(data_iter)
except StopIteration:
# reset data iter
data_iter = iter(dataloader)
sample = next(data_iter)
self.total_train_itrs += 1
self.optimizer.zero_grad()
self.model.train()
total_loss = eval_method(sample)
# compute validation loss
if idx % validation_freq == 0 and val_dataloader is not None:
total_loss_val = []
self.model.eval()
for k, val_sample in enumerate(val_dataloader):
total_loss_val.append(eval_method(val_sample))
if k == val_iters - 1:
total_nll_val = torch.stack(
total_loss_val).mean().detach().numpy()
self.writer.add_scalar('NLL/Val', total_nll_val,
self.train_step)
break
# grad update on logp
total_nll = total_loss
total_nll.backward()
nn.utils.clip_grad_norm_(self.model.parameters(),
config['grad_clip_value'])
self.optimizer.step()
if config['learning_rate_decay']:
self.scheduler.step()
self.model.sigma_scale = max(
1., self.model.sigma_scale * config['sigma_eps_annealing'])
# ---- logging / summaries ------
self.train_step += 1
step = self.train_step
# tensorboard logging
self.writer.add_scalar('NLL/Train', total_nll.item(), step)
if self.model.config['multistep_curriculum']:
curr_step = self.model.config['curriculum_step']
horizon = min(
self.model.config['data_horizon'],
int(np.floor(self.total_train_itrs / curr_step)) + 1)
self.writer.add_scalar('Evaluation_horizon', horizon, step)
Q, Linv = self.model.prior_params()
K = (Linv @ Q.unsqueeze(-1)).squeeze(-1)
_, Linv_sig, _ = torch.svd(Linv)
_, Q_sig, _ = torch.svd(Q)
_, K_sig, _ = torch.svd(K)
for dim in range(config['x_dim']):
self.writer.add_histogram('log_Linv' + str(dim) + '_sig',
torch.log(Linv_sig[dim, :]),
step)
self.writer.add_histogram('Q_sig', Q_sig, step)
self.writer.add_histogram('K_sig', K_sig, step)
self.writer.add_histogram('sigma_eps_val',
self.model.logSigEps, step)
if config['learning_rate_decay']:
self.writer.add_scalar('learning_rate',
self.scheduler.get_lr()[0], step)
self.writer.add_scalar('sigma_eps_scale',
self.model.sigma_scale, step)
self.writer.add_scalar('kernel_bandwidth',
self.model.length_scale, step)
# tqdm logging
logdict = {}
logdict["tr_loss"] = total_nll.cpu().detach().numpy()
if val_dataloader is not None:
logdict["val_loss"] = total_nll_val
if config['learnable_bandwidth']:
logdict["bandwidth"] = self.model.length_scale.item()
pbar.set_postfix(logdict)
self.reset()
def _nullspace(A):
'''Compute the null space of A.
Return the smallest singular value and the corresponding vector.
'''
u, s, vh = torch.svd(A)
return s[..., -1], vh[..., -1]
def pca(x: Tensor, k=2):
"Compute PCA of `x` with `k` dimensions."
x = x - torch.mean(x, 0)
U, S, V = torch.svd(x.t())
return torch.mm(x, U[:, :k])
print(chw_img_data) m = torch.randn(100, 10) v = torch.randn(10) # 内積 d = torch.dot(v, v) print(d, "内積") # 100×10の行列と長さ10のベクトルとの内積，結果は長さ100のベクトル v2 = torch.mv(m, v) print(v2) # 行列積 m2 = torch.mm(m.t(), m) print(m2) # 特異値分解 u, s, v = torch.svd(m) # u，vは直交行列，sは対角成分のみの正方行列．最小二乗法を解く，行列の近似，圧縮 print(u, s, v) x = torch.randn(100, 3) # 微分の変数として扱う場合はrequires_gradフラグをTrueにする a = torch.tensor([1, 2, 3.], requires_grad=True) print(a, "requires_gradフラグをTrue") # 計算をすることで自動的に計算フラグが構築されていく y = torch.mv(x, a) print(y) # 100個 o = y.sum() print(o) # 微分を実行する print(o.backward()) # 解析解と比較 print(a.grad != x.sum(0))
def cluster_loss(H, kloss, lmbda, batch_size):
gram_matrix = (H.T @ H) / batch_size
_, sv_2, _ = torch.svd(gram_matrix)
sv = torch.sqrt(sv_2[:kloss])
loss = torch.sum(sv)
return lmbda * loss
import torch
import time
m = torch.randn(100, 10)
v = torch.randn(10)
print(m.size())
print(v.size())
print('internal product')
d = torch.dot(v, v)
print(d)
print('product')
v2 = torch.mv(m, v)
print(v2.size())
print('matrix product')
m2 = torch.mm(m.t(), m)
print(m2.size())
start = time.time()
print('singular value decomposition')
u, s, v = torch.svd(m)
end = time.time()
print('elapsed_time:{0}'.format(end - start) + '[sec]')
print('with cpu: 0.00026488304138183594[sec]')
def svd_wrapper(matrix, mode, ncomp, debug, verbose, usv=False,
random_state=None, to_numpy=True):
""" Wrapper for different SVD libraries (CPU and GPU).
Parameters
----------
matrix : array_like, 2d
2d input matrix.
mode : {'lapack', 'arpack', 'eigen', 'randsvd', 'cupy', 'eigencupy',
'randcupy', 'pytorch', 'eigenpytorch', 'randpytorch'}, str optional
Switch for the SVD method/library to be used. ``lapack`` uses the LAPACK
linear algebra library through Numpy and it is the most conventional way
of computing the SVD (deterministic result computed on CPU). ``arpack``
uses the ARPACK Fortran libraries accessible through Scipy (computation
on CPU). ``eigen`` computes the singular vectors through the
eigendecomposition of the covariance M.M' (computation on CPU).
``randsvd`` uses the randomized_svd algorithm implemented in Sklearn
(computation on CPU). ``cupy`` uses the Cupy library for GPU computation
of the SVD as in the LAPACK version. ``eigencupy`` offers the same
method as with the ``eigen`` option but on GPU (through Cupy).
``randcupy`` is an adaptation f the randomized_svd algorithm, where all
the computations are done on a GPU (through Cupy). ``pytorch`` uses the
Pytorch library for GPU computation of the SVD. ``eigenpytorch`` offers
the same method as with the ``eigen`` option but on GPU (through
Pytorch). ``randpytorch`` is an adaptation of the randomized_svd
algorithm, where all the linear algebra computations are done on a GPU
(through Pytorch).
ncomp : int
Number of singular vectors to be obtained. In the cases when the full
SVD is computed (LAPACK, ARPACK, EIGEN, CUPY), the matrix of singular
vectors is truncated.
debug : bool
If True the explained variance ratio is computed and displayed.
verbose: bool
If True intermediate information is printed out.
usv : bool optional
If True the 3 terms of the SVD factorization are returned.
random_state : int, RandomState instance or None, optional
If int, random_state is the seed used by the random number generator.
If RandomState instance, random_state is the random number generator.
If None, the random number generator is the RandomState instance used
by np.random. Used for ``randsvd`` mode.
to_numpy : bool, optional
If True (by default) the arrays computed in GPU are transferred from
VRAM and converted to numpy ndarrays.
Returns
-------
V : array_like
The right singular vectors of the input matrix. If ``usv`` is True it
returns the left and right singular vectors and the singular values of
the input matrix.
References
----------
* For ``lapack`` SVD mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.svd.html
http://www.netlib.org/lapack/
* For ``eigen`` mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.eigh.html
* For ``arpack`` SVD mode see:
https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.svds.html
http://www.caam.rice.edu/software/ARPACK/
* For ``randsvd`` SVD mode see:
https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/utils/extmath.py
Finding structure with randomness: Stochastic algorithms for constructing
approximate matrix decompositions
Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061
* For ``cupy`` SVD mode see:
https://docs-cupy.chainer.org/en/stable/reference/generated/cupy.linalg.svd.html
* For ``eigencupy`` mode see:
https://docs-cupy.chainer.org/en/master/reference/generated/cupy.linalg.eigh.html
* For ``pytorch`` SVD mode see:
http://pytorch.org/docs/master/torch.html#torch.svd
* For ``eigenpytorch`` mode see:
http://pytorch.org/docs/master/torch.html#torch.eig
"""
def reconstruction(ncomp, U, S, V, var=1):
if mode == 'lapack':
rec_matrix = np.dot(U[:, :ncomp],
np.dot(np.diag(S[:ncomp]), V[:ncomp]))
rec_matrix = rec_matrix.T
print(' Matrix reconstruction with {} PCs:'.format(ncomp))
print(' Mean Absolute Error =', MAE(matrix, rec_matrix))
print(' Mean Squared Error =', MSE(matrix, rec_matrix))
# see https://github.com/scikit-learn/scikit-learn/blob/c3980bcbabd9d2527548820581725df2904e4a0d/sklearn/decomposition/pca.py
exp_var = (S ** 2) / (S.shape[0] - 1)
full_var = np.sum(exp_var)
explained_variance_ratio = exp_var / full_var # % of variance explained by each PC
ratio_cumsum = np.cumsum(explained_variance_ratio)
elif mode == 'eigen':
exp_var = (S ** 2) / (S.shape[0] - 1)
full_var = np.sum(exp_var)
explained_variance_ratio = exp_var / full_var # % of variance explained by each PC
ratio_cumsum = np.cumsum(explained_variance_ratio)
else:
rec_matrix = np.dot(U, np.dot(np.diag(S), V))
print(' Matrix reconstruction MAE =', MAE(matrix, rec_matrix))
exp_var = (S ** 2) / (S.shape[0] - 1)
full_var = np.var(matrix, axis=0).sum()
explained_variance_ratio = exp_var / full_var # % of variance explained by each PC
if var == 1:
pass
else:
explained_variance_ratio = explained_variance_ratio[::-1]
ratio_cumsum = np.cumsum(explained_variance_ratio)
msg = ' This info makes sense when the matrix is mean centered '
msg += '(temp-mean scaling)'
print(msg)
lw = 2; alpha = 0.4
fig = plt.figure(figsize=vip_figsize)
fig.subplots_adjust(wspace=0.4)
ax1 = plt.subplot2grid((1, 3), (0, 0), colspan=2)
ax1.step(range(explained_variance_ratio.shape[0]),
explained_variance_ratio, alpha=alpha, where='mid',
label='Individual EVR', lw=lw)
ax1.plot(ratio_cumsum, '.-', alpha=alpha,
label='Cumulative EVR', lw=lw)
ax1.legend(loc='best', frameon=False, fontsize='medium')
ax1.set_ylabel('Explained variance ratio (EVR)')
ax1.set_xlabel('Principal components')
ax1.grid(linestyle='solid', alpha=0.2)
ax1.set_xlim(-10, explained_variance_ratio.shape[0] + 10)
ax1.set_ylim(0, 1)
trunc = 20
ax2 = plt.subplot2grid((1, 3), (0, 2), colspan=1)
# plt.setp(ax2.get_yticklabels(), visible=False)
ax2.step(range(trunc), explained_variance_ratio[:trunc], alpha=alpha,
where='mid', lw=lw)
ax2.plot(ratio_cumsum[:trunc], '.-', alpha=alpha, lw=lw)
ax2.set_xlabel('Principal components')
ax2.grid(linestyle='solid', alpha=0.2)
ax2.set_xlim(-2, trunc + 2)
ax2.set_ylim(0, 1)
msg = ' Cumulative explained variance ratio for {} PCs = {:.5f}'
# plt.savefig('figure.pdf', dpi=300, bbox_inches='tight')
print(msg.format(ncomp, ratio_cumsum[ncomp - 1]))
# --------------------------------------------------------------------------
if matrix.ndim != 2:
raise TypeError('Input matrix is not a 2d array')
if usv:
if mode not in ('lapack', 'arpack', 'randsvd', 'cupy', 'randcupy',
'pytorch', 'randpytorch'):
msg = "Returning USV is supported with modes lapack, arpack, "
msg += "randsvd, cupy, randcupy, pytorch or randpytorch"
raise ValueError(msg)
if ncomp > min(matrix.shape[0], matrix.shape[1]):
msg = '{} PCs cannot be obtained from a matrix with size [{},{}].'
msg += ' Increase the size of the patches or request less PCs'
raise RuntimeError(msg.format(ncomp, matrix.shape[0], matrix.shape[1]))
if mode == 'eigen':
# building C as np.dot(matrix.T,matrix) is slower and takes more memory
C = np.dot(matrix, matrix.T) # covariance matrix
e, EV = linalg.eigh(C) # EVals and EVs
pc = np.dot(EV.T, matrix) # PCs using a compact trick when cov is MM'
V = pc[::-1] # reverse since we need the last EVs
S = np.sqrt(np.abs(e)) # SVals = sqrt(EVals)
S = S[::-1] # reverse since EVals go in increasing order
if debug:
reconstruction(ncomp, None, S, None)
for i in range(V.shape[1]):
V[:, i] /= S # scaling EVs by the square root of EVals
V = V[:ncomp]
if verbose:
print('Done PCA with numpy linalg eigh functions')
elif mode == 'lapack':
# n_frames is usually smaller than n_pixels. In this setting taking the SVD of M'
# and keeping the left (transposed) SVs is faster than taking the SVD of M (right SVs)
U, S, V = linalg.svd(matrix.T, full_matrices=False)
if debug:
reconstruction(ncomp, U, S, V)
V = V[:ncomp] # we cut projection matrix according to the # of PCs
U = U[:, :ncomp]
S = S[:ncomp]
if verbose:
print('Done SVD/PCA with numpy SVD (LAPACK)')
elif mode == 'arpack':
U, S, V = svds(matrix, k=ncomp)
if debug:
reconstruction(ncomp, U, S, V, -1)
if verbose:
print('Done SVD/PCA with scipy sparse SVD (ARPACK)')
elif mode == 'randsvd':
U, S, V = randomized_svd(matrix, n_components=ncomp, n_iter=2,
transpose='auto', random_state=random_state)
if debug:
reconstruction(ncomp, U, S, V)
if verbose:
print('Done SVD/PCA with randomized SVD')
elif mode == 'cupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
u_gpu, s_gpu, vh_gpu = cupy.linalg.svd(a_gpu, full_matrices=True,
compute_uv=True)
V = vh_gpu[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if usv:
S = s_gpu[:ncomp]
if to_numpy:
S = cupy.asnumpy(S)
U = u_gpu[:, :ncomp]
if to_numpy:
U = cupy.asnumpy(U)
if verbose:
print('Done SVD/PCA with cupy (GPU)')
elif mode == 'randcupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='cupy')
if to_numpy:
V = cupy.asnumpy(V)
S = cupy.asnumpy(S)
U = cupy.asnumpy(U)
if debug:
reconstruction(ncomp, U, S, V)
if verbose:
print('Done randomized SVD/PCA with cupy (GPU)')
elif mode == 'eigencupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
C = cupy.dot(a_gpu, a_gpu.T) # covariance matrix
e, EV = cupy.linalg.eigh(C) # eigenvalues and eigenvectors
pc = cupy.dot(EV.T, a_gpu) # PCs using a compact trick when cov is MM'
V = pc[::-1] # reverse since last eigenvectors are the ones we want
S = cupy.sqrt(e)[::-1] # reverse since eigenvalues are in increasing order
if debug:
reconstruction(ncomp, None, S, None)
for i in range(V.shape[1]):
V[:, i] /= S # scaling by the square root of eigenvalues
V = V[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if verbose:
print('Done PCA with cupy eigh function (GPU)')
elif mode == 'pytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32').T))
u_gpu, s_gpu, vh_gpu = torch.svd(a_gpu)
V = vh_gpu[:ncomp]
S = s_gpu[:ncomp]
U = torch.transpose(u_gpu, 0, 1)[:ncomp]
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if verbose:
print('Done SVD/PCA with pytorch (GPU)')
elif mode == 'eigenpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32')))
C = torch.mm(a_gpu, torch.transpose(a_gpu, 0, 1))
e, EV = torch.eig(C, eigenvectors=True)
V = torch.mm(torch.transpose(EV, 0, 1), a_gpu)
S = torch.sqrt(e[:, 0])
if debug:
reconstruction(ncomp, None, S, None)
for i in range(V.shape[1]):
V[:, i] /= S
V = V[:ncomp]
if to_numpy:
V = np.array(V)
if verbose:
print('Done PCA with pytorch eig function')
elif mode == 'randpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='pytorch')
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if debug:
reconstruction(ncomp, U, S, V)
if verbose:
print('Done randomized SVD/PCA with randomized pytorch (GPU)')
else:
raise ValueError('The SVD mode is not available')
if usv:
if mode == 'lapack':
return V.T, S, U.T
elif mode == 'pytorch':
if to_numpy:
return V.T, S, U.T
else:
return torch.transpose(V, 0, 1), S, torch.transpose(U, 0, 1)
else:
return U, S, V
else:
if mode == 'lapack':
return U.T
elif mode == 'pytorch':
return U
else:
return V
def forward(self, features):
# start_time = time.time()
# features: NCWH
k = features.size(0) * features.size(2) * features.size(3)
x_mean = (features.sum(dim=2).sum(dim=2).sum(dim=0) /
k).unsqueeze(0).unsqueeze(2).unsqueeze(2)
# print("1--- %s seconds ---" % (time.time() - start_time))
# d = time.time()
features = features - x_mean
# print("2--- %s seconds ---" % (time.time() - d))
# d = time.time()
reshaped_features = features.view(features.size(0), features.size(1), -1)\
.permute(1, 0, 2).contiguous().view(features.size(1), -1)
cov = torch.matmul(reshaped_features, reshaped_features.t()) / k
# print("3--- %s seconds ---" % (time.time() - d))
# d = time.time()
# print('cov',cov.shape)
# eigval, eigvec = torch.eig(cov, eigenvectors=True)
# print("4--- %s seconds ---" % (time.time() - d))
d = time.time()
_, _, eigvec = torch.svd(cov)
# print("4s--- %s seconds ---" % (time.time() - d))
# d = time.time()
first_compo = eigvec[:, 0]
first_projected_map = torch.matmul(first_compo.unsqueeze(0), reshaped_features).view(1, features.size(0), -1)\
.view(features.size(0), features.size(2), features.size(3))
# print("5--- %s seconds ---" % (time.time() - d))
# d = time.time()
first_maxv = first_projected_map.max()
first_minv = first_projected_map.min()
first_projected_map *= (
first_maxv + first_minv) / torch.abs(first_maxv + first_minv)
# print("6--- %s seconds ---" % (time.time() - d))
# d = time.time()
second_compo = eigvec[:, 1]
second_projected_map = torch.matmul(second_compo.unsqueeze(0), reshaped_features).view(1, features.size(0), -1)\
.view(features.size(0), features.size(2), features.size(3))
second_maxv = second_projected_map.max()
second_minv = second_projected_map.min()
second_projected_map *= (
second_maxv + second_minv) / torch.abs(second_maxv + second_minv)
third_compo = eigvec[:, 2]
third_projected_map = torch.matmul(third_compo.unsqueeze(0), reshaped_features).view(1, features.size(0), -1)\
.view(features.size(0), features.size(2), features.size(3))
third_maxv = third_projected_map.max()
third_minv = third_projected_map.min()
third_projected_map *= (
third_maxv + third_minv) / torch.abs(third_maxv + third_minv)
# print("7--- %s seconds ---" % (time.time() - d))
# d = time.time()
return [first_projected_map, second_projected_map, third_projected_map]
def svd_wrapper(matrix, mode, ncomp, verbose, full_output=False,
random_state=None, to_numpy=True):
""" Wrapper for different SVD libraries (CPU and GPU).
Parameters
----------
matrix : numpy ndarray, 2d
2d input matrix.
mode : {'lapack', 'arpack', 'eigen', 'randsvd', 'cupy', 'eigencupy',
'randcupy', 'pytorch', 'eigenpytorch', 'randpytorch'}, str optional
Switch for the SVD method/library to be used.
``lapack``: uses the LAPACK linear algebra library through Numpy
and it is the most conventional way of computing the SVD
(deterministic result computed on CPU).
``arpack``: uses the ARPACK Fortran libraries accessible through
Scipy (computation on CPU).
``eigen``: computes the singular vectors through the
eigendecomposition of the covariance M.M' (computation on CPU).
``randsvd``: uses the randomized_svd algorithm implemented in
Sklearn (computation on CPU).
``cupy``: uses the Cupy library for GPU computation of the SVD as in
the LAPACK version. `
`eigencupy``: offers the same method as with the ``eigen`` option
but on GPU (through Cupy).
``randcupy``: is an adaptation of the randomized_svd algorithm,
where all the computations are done on a GPU (through Cupy). `
`pytorch``: uses the Pytorch library for GPU computation of the SVD.
``eigenpytorch``: offers the same method as with the ``eigen``
option but on GPU (through Pytorch).
``randpytorch``: is an adaptation of the randomized_svd algorithm,
where all the linear algebra computations are done on a GPU
(through Pytorch).
ncomp : int
Number of singular vectors to be obtained. In the cases when the full
SVD is computed (LAPACK, ARPACK, EIGEN, CUPY), the matrix of singular
vectors is truncated.
verbose: bool
If True intermediate information is printed out.
full_output : bool optional
If True the 3 terms of the SVD factorization are returned. If ``mode``
is eigen then only S and V are returned.
random_state : int, RandomState instance or None, optional
If int, random_state is the seed used by the random number generator.
If RandomState instance, random_state is the random number generator.
If None, the random number generator is the RandomState instance used
by np.random. Used for ``randsvd`` mode.
to_numpy : bool, optional
If True (by default) the arrays computed in GPU are transferred from
VRAM and converted to numpy ndarrays.
Returns
-------
V : numpy ndarray
The right singular vectors of the input matrix. If ``full_output`` is
True it returns the left and right singular vectors and the singular
values of the input matrix. If ``mode`` is set to eigen then only S and
V are returned.
References
----------
* For ``lapack`` SVD mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.svd.html
http://www.netlib.org/lapack/
* For ``eigen`` mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.eigh.html
* For ``arpack`` SVD mode see:
https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.svds.html
http://www.caam.rice.edu/software/ARPACK/
* For ``randsvd`` SVD mode see:
https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/utils/extmath.py
Finding structure with randomness: Stochastic algorithms for constructing
approximate matrix decompositions
Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061
* For ``cupy`` SVD mode see:
https://docs-cupy.chainer.org/en/stable/reference/generated/cupy.linalg.svd.html
* For ``eigencupy`` mode see:
https://docs-cupy.chainer.org/en/master/reference/generated/cupy.linalg.eigh.html
* For ``pytorch`` SVD mode see:
http://pytorch.org/docs/master/torch.html#torch.svd
* For ``eigenpytorch`` mode see:
http://pytorch.org/docs/master/torch.html#torch.eig
"""
if matrix.ndim != 2:
raise TypeError('Input matrix is not a 2d array')
if ncomp > min(matrix.shape[0], matrix.shape[1]):
msg = '{} PCs cannot be obtained from a matrix with size [{},{}].'
msg += ' Increase the size of the patches or request less PCs'
raise RuntimeError(msg.format(ncomp, matrix.shape[0], matrix.shape[1]))
if mode == 'eigen':
# building C as np.dot(matrix.T,matrix) is slower and takes more memory
C = np.dot(matrix, matrix.T) # covariance matrix
e, EV = linalg.eigh(C) # EVals and EVs
pc = np.dot(EV.T, matrix) # PCs using a compact trick when cov is MM'
V = pc[::-1] # reverse since we need the last EVs
S = np.sqrt(np.abs(e)) # SVals = sqrt(EVals)
S = S[::-1] # reverse since EVals go in increasing order
for i in range(V.shape[1]):
V[:, i] /= S # scaling EVs by the square root of EVals
V = V[:ncomp]
if verbose:
print('Done PCA with numpy linalg eigh functions')
elif mode == 'lapack':
# n_frames is usually smaller than n_pixels. In this setting taking
# the SVD of M' and keeping the left (transposed) SVs is faster than
# taking the SVD of M (right SVs)
U, S, V = linalg.svd(matrix.T, full_matrices=False)
V = V[:ncomp] # we cut projection matrix according to the # of PCs
U = U[:, :ncomp]
S = S[:ncomp]
if verbose:
print('Done SVD/PCA with numpy SVD (LAPACK)')
elif mode == 'arpack':
U, S, V = svds(matrix, k=ncomp)
if verbose:
print('Done SVD/PCA with scipy sparse SVD (ARPACK)')
elif mode == 'randsvd':
U, S, V = randomized_svd(matrix, n_components=ncomp, n_iter=2,
transpose='auto', random_state=random_state)
if verbose:
print('Done SVD/PCA with randomized SVD')
elif mode == 'cupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
u_gpu, s_gpu, vh_gpu = cupy.linalg.svd(a_gpu, full_matrices=True,
compute_uv=True)
V = vh_gpu[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if full_output:
S = s_gpu[:ncomp]
if to_numpy:
S = cupy.asnumpy(S)
U = u_gpu[:, :ncomp]
if to_numpy:
U = cupy.asnumpy(U)
if verbose:
print('Done SVD/PCA with cupy (GPU)')
elif mode == 'randcupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='cupy')
if to_numpy:
V = cupy.asnumpy(V)
S = cupy.asnumpy(S)
U = cupy.asnumpy(U)
if verbose:
print('Done randomized SVD/PCA with cupy (GPU)')
elif mode == 'eigencupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
C = cupy.dot(a_gpu, a_gpu.T) # covariance matrix
e, EV = cupy.linalg.eigh(C) # eigenvalues and eigenvectors
pc = cupy.dot(EV.T, a_gpu) # using a compact trick when cov is MM'
V = pc[::-1] # reverse to get last eigenvectors
S = cupy.sqrt(e)[::-1] # reverse since EVals go in increasing order
for i in range(V.shape[1]):
V[:, i] /= S # scaling by the square root of eigvals
V = V[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if verbose:
print('Done PCA with cupy eigh function (GPU)')
elif mode == 'pytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32').T))
u_gpu, s_gpu, vh_gpu = torch.svd(a_gpu)
V = vh_gpu[:ncomp]
S = s_gpu[:ncomp]
U = torch.transpose(u_gpu, 0, 1)[:ncomp]
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if verbose:
print('Done SVD/PCA with pytorch (GPU)')
elif mode == 'eigenpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32')))
C = torch.mm(a_gpu, torch.transpose(a_gpu, 0, 1))
e, EV = torch.eig(C, eigenvectors=True)
V = torch.mm(torch.transpose(EV, 0, 1), a_gpu)
S = torch.sqrt(e[:, 0])
for i in range(V.shape[1]):
V[:, i] /= S
V = V[:ncomp]
if to_numpy:
V = np.array(V)
if verbose:
print('Done PCA with pytorch eig function')
elif mode == 'randpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='pytorch')
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if verbose:
print('Done randomized SVD/PCA with randomized pytorch (GPU)')
else:
raise ValueError('The SVD `mode` is not recognized')
if full_output:
if mode == 'lapack':
return V.T, S, U.T
elif mode == 'pytorch':
if to_numpy:
return V.T, S, U.T
else:
return torch.transpose(V, 0, 1), S, torch.transpose(U, 0, 1)
elif mode in ('eigen', 'eigencupy', 'eigenpytorch'):
return S, V
else:
return U, S, V
else:
if mode == 'lapack':
return U.T
elif mode == 'pytorch':
return U
else:
return V
def EVBMF(Y, sigma2=None, H=None):
"""Implementation of the analytical solution to Empirical Variational Bayes Matrix Factorization.
This function can be used to calculate the analytical solution to empirical VBMF.
This is based on the paper and MatLab code by Nakajima et al.:
"Global analytic solution of fully-observed variational Bayesian matrix factorization."
Notes
-----
If sigma2 is unspecified, it is estimated by minimizing the free energy.
If H is unspecified, it is set to the smallest of the sides of the input Y.
Attributes
----------
Y : numpy-array
Input matrix that is to be factorized. Y has shape (L,M), where L<=M.
sigma2 : int or None (default=None)
Variance of the noise on Y.
H : int or None (default = None)
Maximum rank of the factorized matrices.
Returns
-------
U : numpy-array
Left-singular vectors.
S : numpy-array
Diagonal matrix of singular values.
V : numpy-array
Right-singular vectors.
post : dictionary
Dictionary containing the computed posterior values.
References
----------
.. [1] Nakajima, Shinichi, et al. "Global analytic solution of fully-observed variational Bayesian matrix factorization." Journal of Machine Learning Research 14.Jan (2013): 1-37.
.. [2] Nakajima, Shinichi, et al. "Perfect dimensionality recovery by variational Bayesian PCA." Advances in Neural Information Processing Systems. 2012.
"""
L, M = Y.shape # has to be L<=M
if H is None:
H = L
alpha = L / M
tauubar = 2.5129 * np.sqrt(alpha)
# SVD of the input matrix, max rank of H
U, s, V = torch.svd(Y)
U = U[:, :H]
s = s[:H]
V[:H].t_()
# Calculate residual
residual = 0.
if H < L:
residual = torch.sum(torch.sum(Y**2) - torch.sum(s**2))
# Estimation of the variance when sigma2 is unspecified
if sigma2 is None:
xubar = (1 + tauubar) * (1 + alpha / tauubar)
eH_ub = int(np.min([np.ceil(L / (1 + alpha)) - 1, H])) - 1
upper_bound = (torch.sum(s**2) + residual) / (L * M)
upper_bound = upper_bound.item()
lower_bound = np.max(
[s[eH_ub + 1]**2 / (M * xubar),
torch.mean(s[eH_ub + 1:]**2) / M])
scale = 1. # /lower_bound
s = s * np.sqrt(scale)
residual = residual * scale
lower_bound = lower_bound * scale
upper_bound = upper_bound * scale
sigma2_opt = minimize_scalar(EVBsigma2,
args=(L, M, s, residual, xubar),
bounds=[lower_bound, upper_bound],
method='Bounded')
sigma2 = sigma2_opt.x
# Threshold gamma term
threshold = np.sqrt(M * sigma2 * (1 + tauubar) * (1 + alpha / tauubar))
pos = torch.sum(s > threshold)
if pos == 0:
return np.array([])
# Formula (15) from [2]
d = torch.mul(
s[:pos] / 2, 1 - (L + M) * sigma2 / s[:pos]**2 +
torch.sqrt((1 - ((L + M) * sigma2) / s[:pos]**2)**2 -
(4 * L * M * sigma2**2) / s[:pos]**4))
return torch.diag(d)
def main():
cfg.configure(args)
cfg.print_config()
torch.set_num_threads(args.omp_cores)
torch.manual_seed(args.seed)
if args.c4v_type == "TI":
model = jq.JQ_C4V(j1=args.j1, q=args.q)
elif args.c4v_type == "BIPARTITE":
model = jq.JQ_C4V_BIPARTITE(j1=args.j1, q=args.q)
elif args.c4v_type == "PLAQUETTE":
q_inter = args.q if args.q_inter is None else args.q_inter
model = jq.JQ_C4V_PLAQUETTE(j1=args.j1, q=args.q, q_inter=q_inter)
else:
raise ValueError("Unsupported C4v ansatz: -c4v_type= "\
+str(args.ipeps_init_type)+" is not supported")
# initialize an ipeps
# 1) define lattice-tiling function, that maps arbitrary vertex of square lattice
# coord into one of coordinates within unit-cell of iPEPS ansatz
if args.instate != None:
state = read_ipeps_c4v(args.instate)
if args.bond_dim > max(state.get_aux_bond_dims()):
# extend the auxiliary dimensions
state = extend_bond_dim(state, args.bond_dim)
state.add_noise(args.instate_noise)
state.sites[(0, 0)] = state.sites[(0, 0)] / torch.max(
torch.abs(state.sites[(0, 0)]))
elif args.ipeps_init_type == 'RANDOM':
bond_dim = args.bond_dim
A= torch.rand((model.phys_dim, bond_dim, bond_dim, bond_dim, bond_dim),\
dtype=cfg.global_args.dtype,device=cfg.global_args.device)
A = make_c4v_symm(A)
A = A / torch.max(torch.abs(A))
state = IPEPS_C4V(A)
else:
raise ValueError("Missing trial state: -instate=None and -ipeps_init_type= "\
+str(args.ipeps_init_type)+" is not supported")
print(state)
def ctmrg_conv_energy(state, env, history, ctm_args=cfg.ctm_args):
with torch.no_grad():
if not history:
history = []
e_curr = model.energy_1x1(state, env)
obs_values, obs_labels = model.eval_obs(state, env)
history.append([e_curr.item()] + obs_values)
print(", ".join([f"{len(history)}", f"{e_curr}"] +
[f"{v}" for v in obs_values]))
if len(history) > 1 and abs(
history[-1][0] - history[-2][0]) < ctm_args.ctm_conv_tol:
return True, history
return False, history
ctm_env_init = ENV_C4V(args.chi, state)
init_env(state, ctm_env_init)
print(ctm_env_init)
e_curr0 = model.energy_1x1(state, ctm_env_init)
obs_values0, obs_labels = model.eval_obs(state, ctm_env_init)
print(", ".join(["epoch", "energy"] + obs_labels))
print(", ".join([f"{-1}", f"{e_curr0}"] + [f"{v}" for v in obs_values0]))
ctm_env_init, *ctm_log = ctmrg_c4v.run(state,
ctm_env_init,
conv_check=ctmrg_conv_energy)
corrSS = model.eval_corrf_SS(state, ctm_env_init, args.corrf_r)
print("\n\nSS r " + " ".join([label for label in corrSS.keys()]))
for i in range(args.corrf_r):
print(f"{i} " +
" ".join([f"{corrSS[label][i]}" for label in corrSS.keys()]))
corrDD = model.eval_corrf_DD_H(state, ctm_env_init, args.corrf_r)
print("\n\nDD r " + " ".join([label for label in corrDD.keys()]))
for i in range(args.corrf_r):
print(f"{i} " +
" ".join([f"{corrDD[label][i]}" for label in corrDD.keys()]))
corrDD_V = model.eval_corrf_DD_V(state, ctm_env_init, args.corrf_r)
print("\n\nDD_V r " + " ".join([label for label in corrDD_V.keys()]))
for i in range(args.corrf_r):
print(f"{i} " +
" ".join([f"{corrDD_V[label][i]}" for label in corrDD_V.keys()]))
# environment diagnostics
print("\n\nspectrum(C)")
u, s, v = torch.svd(ctm_env_init.C[ctm_env_init.keyC], compute_uv=False)
for i in range(args.chi):
print(f"{i} {s[i]}")
# transfer operator spectrum
print("\n\nspectrum(T)")
l = transferops_c4v.get_Top_spec_c4v(args.top_n, state, ctm_env_init)
for i in range(l.size()[0]):
print(f"{i} {l[i,0]} {l[i,1]}")
def show_low_rank(model,
look_up_table=[],
input_size=None,
criterion=None,
type='NC'):
redundancy = OrderedDict()
comp_rate = OrderedDict()
if input_size is not None:
if isinstance(input_size, int):
input_size = [input_size, input_size]
elif isinstance(input_size, list):
pass
else:
raise Exception
if criterion is None:
raise Exception('criterion must be set for sigma selection')
if input_size is None:
raise Exception('invalid input size')
origin_FLOPs = 0.
decouple_FLOPs = 0.
channels_ = []
for name, m in model.named_modules():
if not isinstance(m, nn.Conv2d):
continue
p = m.weight.data
dim = p.size()
FLOPs = dim[0] * dim[1] * dim[2] * dim[3]
if name in look_up_table:
if args.stride_1_only:
print('Skipping those not with stride 1')
if not m.stride == (1, 1):
continue
if type == 'NC':
NC = p.view(dim[0], -1)
N, sigma, C = torch.svd(NC, some=True)
item_num = criterion(sigma)
print(item_num)
channels_.append(item_num)
new_FLOPs = dim[1] * dim[2] * dim[
3] * item_num + item_num * dim[0]
elif type == 'VH':
VH = p.permute(1, 2, 0,
3).contiguous().view(dim[1] * dim[2], -1)
V, sigma, H = torch.svd(VH, some=True)
item_num = criterion(sigma)
new_FLOPs = dim[1] * item_num * dim[2] + dim[
0] * item_num * dim[3]
else:
valid_idx = []
for i in range(dim[0]):
W = p[i, :, :, :].view(dim[1], -1)
U, sigma, V = torch.svd(W, some=True)
valid_idx.append(criterion(sigma))
item_num = min(max(valid_idx), min(dim[1], dim[2] * dim[3]))
new_FLOPs = (dim[0] * dim[1] +
dim[0] * dim[2] * dim[3]) * item_num
rate = float(new_FLOPs) / FLOPs
comp_rate[name] = ('%.3f' % (rate))
else:
new_FLOPs = FLOPs
if 'downsample' not in name:
# a special case for resnet
output_h = input_size[0] / m.stride[0]
output_w = input_size[1] / m.stride[1]
else:
output_h = input_size[0]
output_w = input_size[1]
origin_FLOPs += FLOPs * output_h * output_w
decouple_FLOPs += new_FLOPs * output_h * output_w
input_size = [output_h, output_w]
r = origin_FLOPs / decouple_FLOPs
if DEBUG:
print(comp_rate)
print('\n')
print('comp rate:')
print(r)
return r, channels_
x_torch = utils.convert_into_at_least_2d_pytorch_tensor(
x).requires_grad_().to(self.device)
J_torch = self.J_torch(x_torch)
return J_torch.cpu().detach().numpy()
if __name__ == "__main__":
rand_seed = 47
np.random.seed(rand_seed)
torch.random.manual_seed(rand_seed)
dfeat = DummyDifferentiableFeature()
x = np.random.rand(2, 5)
print("x = ")
print(x)
J = dfeat.J(x)
print("J = ")
print(J)
J_torch = utils.convert_into_at_least_2d_pytorch_tensor(
J).requires_grad_().to('cpu')
[J_svd_U_torch, J_svd_s_torch, J_svd_V_torch] = torch.svd(J_torch,
some=True)
print("J_svd_U_torch = ", J_svd_U_torch)
print("J_svd_s_torch = ", J_svd_s_torch)
print("J_svd_V_torch = ", J_svd_V_torch)
# print("J_svd_U_torch @ J_svd_U_torch.transpose() = ", J_svd_U_torch @ J_svd_U_torch.transpose(-2, -1))
# print("J_svd_U_torch.transpose() @ J_svd_U_torch = ", J_svd_U_torch.transpose(-2, -1) @ J_svd_U_torch)
# print("J_svd_V_torch @ J_svd_V_torch.transpose() = ", J_svd_V_torch @ J_svd_V_torch.transpose(-2, -1))
# print("J_svd_V_torch.transpose() @ J_svd_V_torch = ", J_svd_V_torch.transpose(-2, -1) @ J_svd_V_torch)
# print("det(J_svd_U_torch) = ", torch.det(J_svd_U_torch))
def low_rank_approx(model, look_up_table, criterion, use_trp, type='NC'):
dict2 = model.state_dict()
sub = dict()
#can set m here
for name in dict2:
param = dict2[name]
dim = param.size()
model_name = name[:-7] if len(dim) == 4 else ''
if len(dim) == 4 and model_name in look_up_table:
if type == 'VH':
VH = param.permute(1, 2, 0,
3).contiguous().view(dim[1] * dim[2], -1)
try:
V, sigma, H = torch.svd(VH, some=True)
# print(sigma.size())
H = H.t()
# remain large singular value
valid_idx = criterion(sigma)
V = V[:, :valid_idx].contiguous()
sigma = sigma[:valid_idx]
dia = torch.diag(sigma)
H = H[:valid_idx, :]
if use_trp:
new_VH = (V.mm(dia)).mm(H)
new_VH = new_VH.contiguous().view(
dim[1], dim[2], dim[0],
dim[3]).permute(2, 0, 1, 3)
dict2[name].copy_(new_VH)
subgradient = torch.mm(V, H)
subgradient = subgradient.contiguous().view(
dim[1], dim[2], dim[0], dim[3]).permute(2, 0, 1, 3)
sub[model_name] = subgradient
except:
sub[model_name] = 0.0
dict2[name].copy_(param)
elif type == 'NC':
NC = param.contiguous().view(dim[0], -1)
try:
N, sigma, C = torch.svd(NC, some=True)
# print(sigma.size())
C = C.t()
# remain large singular value
valid_idx = criterion(sigma)
N = N[:, :valid_idx].contiguous()
sigma = sigma[:valid_idx]
dia = torch.diag(sigma)
C = C[:valid_idx, :]
if use_trp:
new_NC = (N.mm(dia)).mm(C)
new_NC = new_NC.contiguous().view(
dim[0], dim[1], dim[2], dim[3])
dict2[name].copy_(new_NC)
# this the gradient step
subgradient = torch.mm(N, C)
subgradient = subgradient.contiguous().view(
dim[0], dim[1], dim[2], dim[3])
sub[model_name] = subgradient
except:
sub[model_name] = 0.0
dict2[name].copy_(param)
else:
# network decouple approximation
tmp = param.clone()
tmp_sub = param.clone()
valid_idx = 0
for i in range(dim[0]):
W = param[i, :, :, :].view(dim[1], -1)
try:
U, sigma, V = torch.svd(W, some=True)
V = V.t()
valid_idx = criterion(sigma)
U = U[:, :valid_idx].contiguous()
V = V[:valid_idx, :].contiguous()
sigma = sigma[:valid_idx]
dia = torch.diag(sigma)
if use_trp:
new_W = (U.mm(dia)).mm(V)
new_W = new_W.contiguous().view(
dim[1], dim[2], dim[3])
tmp[i, :, :, :] = new_W[...]
subgradient = torch.mm(U, V)
subgradient = subgradient.contiguous().view(
dim[1], dim[2], dim[3])
tmp_sub[i, :, :, :] = subgradient[...]
except Exception as e:
print(e)
tmp_sub[i, :, :, :] = 0.0
tmp[i, :, :, :] = param[i, :, :, :]
dict2[name].copy_(tmp)
sub[model_name] = tmp_sub
else:
dict2[name].copy_(param)
model.load_state_dict(dict2)
return model, sub
torch.manual_seed(0)
with torch.no_grad():
print('Generating teacher outputs')
for i, (images, _) in enumerate(trainLoader):
images = Variable(images)
images = images.cuda()
_, student_activation = model.forward(images, True)
curr_output, curr_activations = teacherModel.forward(images, True)
student_activations.append(student_activation)
teacher_outputs.append(curr_output)
teacher_activations.append(curr_activations)
student_activations = torch.cat(student_activations)
teacher_outputs = torch.cat(teacher_outputs)
teacher_activations = torch.cat(teacher_activations)
train_svd = torch.svd(teacher_activations)
teacher_Us = train_svd.U
# we use eos_scale to scale the EOS loss during training, so that the same
# learning rates that were good for MSE loss are also good for EOS loss.
eos_scale = torch.norm(teacher_activations) / torch.norm(teacher_Us)
print('Done!')
# train!
torch.manual_seed(0)
bestAcc = 0.0
for epoch in range(1, args.epochs):
train(epoch)
acc = eval_training(epoch)
if epoch > args.warm:
train_scheduler.step()
def spectral_normalize(tensor: Tensor) -> Tensor:
u, s, v = torch.svd(tensor, compute_uv=False)
norm = torch.max(s)
return tensor / norm
def func(): torch.svd(torch.rand((N, N)))
def procrustes(X_source, X_target):
"""min |X_source Q - X_target|_F s.t. Q^TQ = I"""
U, _, V = torch.svd(X_target.T @ X_source)
return V @ U.T
def reset_parameters(self):
torch.nn.init.xavier_uniform_(self.A)
u, s, v = torch.svd(self.A)
self.A.data = u.matmul(v.t())
def SubspaceNetHead(query,
support,
support_labels,
n_way,
n_shot,
normalize=True):
"""
Constructs the subspace representation of each class(=mean of support vectors of each class) and
returns the classification score (=L2 distance to each class prototype) on the query set.
Our algorithm using subspaces here
Parameters:
query: a (tasks_per_batch, n_query, d) Tensor.
support: a (tasks_per_batch, n_support, d) Tensor.
support_labels: a (tasks_per_batch, n_support) Tensor.
n_way: a scalar. Represents the number of classes in a few-shot classification task.
n_shot: a scalar. Represents the number of support examples given per class.
normalize: a boolean. Represents whether if we want to normalize the distances by the embedding dimension.
Returns: a (tasks_per_batch, n_query, n_way) Tensor.
"""
tasks_per_batch = query.size(0)
n_support = support.size(1)
n_query = query.size(1)
d = query.size(2)
assert (query.dim() == 3)
assert (support.dim() == 3)
assert (query.size(0) == support.size(0)
and query.size(2) == support.size(2))
assert (n_support == n_way * n_shot
) # n_support must equal to n_way * n_shot
support_labels_one_hot = one_hot(
support_labels.view(tasks_per_batch * n_support), n_way)
#support_labels_one_hot = support_labels_one_hot.view(tasks_per_batch, n_support, n_way)
support_reshape = support.view(tasks_per_batch * n_support, -1)
support_labels_reshaped = support_labels.contiguous().view(-1)
class_representatives = []
for nn in range(n_way):
idxss = (support_labels_reshaped == nn).nonzero()
all_support_perclass = support_reshape[idxss, :]
class_representatives.append(
all_support_perclass.view(tasks_per_batch, n_shot, -1))
class_representatives = torch.stack(class_representatives)
class_representatives = class_representatives.transpose(
0, 1) #tasks_per_batch, n_way, n_support, -1
class_representatives = class_representatives.transpose(
2, 3).contiguous().view(tasks_per_batch * n_way, -1, n_shot)
dist = []
for cc in range(tasks_per_batch * n_way):
batch_idx = cc // n_way
qq = query[batch_idx]
uu, _, _ = torch.svd(class_representatives[cc].double())
uu = uu.float()
subspace = uu[:, :n_shot - 1].transpose(0, 1)
projection = subspace.transpose(0, 1).mm(
subspace.mm(qq.transpose(0, 1))).transpose(0, 1)
dist_perclass = torch.sum((qq - projection)**2, dim=-1)
dist.append(dist_perclass)
dist = torch.stack(dist).view(tasks_per_batch, n_way, -1).transpose(1, 2)
logits = -dist
if normalize:
logits = logits / d
return logits
def svd_wrapper(matrix,
mode,
ncomp,
debug,
verbose,
usv=False,
random_state=None,
to_numpy=True):
""" Wrapper for different SVD libraries (CPU and GPU).
Parameters
----------
matrix : array_like, 2d
2d input matrix.
mode : {'lapack', 'arpack', 'eigen', 'randsvd', 'cupy', 'eigencupy',
'randcupy', 'pytorch', 'eigenpytorch', 'randpytorch'}, str optional
Switch for the SVD method/library to be used. ``lapack`` uses the LAPACK
linear algebra library through Numpy and it is the most conventional way
of computing the SVD (deterministic result computed on CPU). ``arpack``
uses the ARPACK Fortran libraries accessible through Scipy (computation
on CPU). ``eigen`` computes the singular vectors through the
eigendecomposition of the covariance M.M' (computation on CPU).
``randsvd`` uses the randomized_svd algorithm implemented in Sklearn
(computation on CPU). ``cupy`` uses the Cupy library for GPU computation
of the SVD as in the LAPACK version. ``eigencupy`` offers the same
method as with the ``eigen`` option but on GPU (through Cupy).
``randcupy`` is an adaptation f the randomized_svd algorithm, where all
the computations are done on a GPU (through Cupy). ``pytorch`` uses the
Pytorch library for GPU computation of the SVD. ``eigenpytorch`` offers
the same method as with the ``eigen`` option but on GPU (through
Pytorch). ``randpytorch`` is an adaptation of the randomized_svd
algorithm, where all the linear algebra computations are done on a GPU
(through Pytorch).
ncomp : int
Number of singular vectors to be obtained. In the cases when the full
SVD is computed (LAPACK, ARPACK, EIGEN, CUPY), the matrix of singular
vectors is truncated.
debug : bool
If True the explained variance ratio is computed and displayed.
verbose: bool
If True intermediate information is printed out.
usv : bool optional
If True the 3 terms of the SVD factorization are returned.
random_state : int, RandomState instance or None, optional
If int, random_state is the seed used by the random number generator.
If RandomState instance, random_state is the random number generator.
If None, the random number generator is the RandomState instance used
by np.random. Used for ``randsvd`` mode.
to_numpy : bool, optional
If True (by default) the arrays computed in GPU are transferred from
VRAM and converted to numpy ndarrays.
Returns
-------
V : array_like
The right singular vectors of the input matrix. If ``usv`` is True it
returns the left and right singular vectors and the singular values of
the input matrix.
References
----------
* For ``lapack`` SVD mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.svd.html
http://www.netlib.org/lapack/
* For ``eigen`` mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.eigh.html
* For ``arpack`` SVD mode see:
https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.svds.html
http://www.caam.rice.edu/software/ARPACK/
* For ``randsvd`` SVD mode see:
https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/utils/extmath.py
Finding structure with randomness: Stochastic algorithms for constructing
approximate matrix decompositions
Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061
* For ``cupy`` SVD mode see:
https://docs-cupy.chainer.org/en/stable/reference/generated/cupy.linalg.svd.html
* For ``eigencupy`` mode see:
https://docs-cupy.chainer.org/en/master/reference/generated/cupy.linalg.eigh.html
* For ``pytorch`` SVD mode see:
http://pytorch.org/docs/master/torch.html#torch.svd
* For ``eigenpytorch`` mode see:
http://pytorch.org/docs/master/torch.html#torch.eig
"""
def reconstruction(ncomp, U, S, V, var=1):
if mode == 'lapack':
rec_matrix = np.dot(U[:, :ncomp],
np.dot(np.diag(S[:ncomp]), V[:ncomp]))
rec_matrix = rec_matrix.T
print(' Matrix reconstruction with {} PCs:'.format(ncomp))
print(' Mean Absolute Error =', MAE(matrix, rec_matrix))
print(' Mean Squared Error =', MSE(matrix, rec_matrix))
# see https://github.com/scikit-learn/scikit-learn/blob/c3980bcbabd9d2527548820581725df2904e4a0d/sklearn/decomposition/pca.py
exp_var = (S**2) / (S.shape[0] - 1)
full_var = np.sum(exp_var)
explained_variance_ratio = exp_var / full_var # % of variance explained by each PC
ratio_cumsum = np.cumsum(explained_variance_ratio)
elif mode == 'eigen':
exp_var = (S**2) / (S.shape[0] - 1)
full_var = np.sum(exp_var)
explained_variance_ratio = exp_var / full_var # % of variance explained by each PC
ratio_cumsum = np.cumsum(explained_variance_ratio)
else:
rec_matrix = np.dot(U, np.dot(np.diag(S), V))
print(' Matrix reconstruction MAE =', MAE(matrix, rec_matrix))
exp_var = (S**2) / (S.shape[0] - 1)
full_var = np.var(matrix, axis=0).sum()
explained_variance_ratio = exp_var / full_var # % of variance explained by each PC
if var == 1:
pass
else:
explained_variance_ratio = explained_variance_ratio[::-1]
ratio_cumsum = np.cumsum(explained_variance_ratio)
msg = ' This info makes sense when the matrix is mean centered '
msg += '(temp-mean scaling)'
print(msg)
lw = 2
alpha = 0.4
fig = plt.figure(figsize=vip_figsize)
fig.subplots_adjust(wspace=0.4)
ax1 = plt.subplot2grid((1, 3), (0, 0), colspan=2)
ax1.step(range(explained_variance_ratio.shape[0]),
explained_variance_ratio,
alpha=alpha,
where='mid',
label='Individual EVR',
lw=lw)
ax1.plot(ratio_cumsum,
'.-',
alpha=alpha,
label='Cumulative EVR',
lw=lw)
ax1.legend(loc='best', frameon=False, fontsize='medium')
ax1.set_ylabel('Explained variance ratio (EVR)')
ax1.set_xlabel('Principal components')
ax1.grid(linestyle='solid', alpha=0.2)
ax1.set_xlim(-10, explained_variance_ratio.shape[0] + 10)
ax1.set_ylim(0, 1)
trunc = 20
ax2 = plt.subplot2grid((1, 3), (0, 2), colspan=1)
# plt.setp(ax2.get_yticklabels(), visible=False)
ax2.step(range(trunc),
explained_variance_ratio[:trunc],
alpha=alpha,
where='mid',
lw=lw)
ax2.plot(ratio_cumsum[:trunc], '.-', alpha=alpha, lw=lw)
ax2.set_xlabel('Principal components')
ax2.grid(linestyle='solid', alpha=0.2)
ax2.set_xlim(-2, trunc + 2)
ax2.set_ylim(0, 1)
msg = ' Cumulative explained variance ratio for {} PCs = {:.5f}'
# plt.savefig('figure.pdf', dpi=300, bbox_inches='tight')
print(msg.format(ncomp, ratio_cumsum[ncomp - 1]))
# --------------------------------------------------------------------------
if matrix.ndim != 2:
raise TypeError('Input matrix is not a 2d array')
if usv:
if mode not in ('lapack', 'arpack', 'randsvd', 'cupy', 'randcupy',
'pytorch', 'randpytorch'):
msg = "Returning USV is supported with modes lapack, arpack, "
msg += "randsvd, cupy, randcupy, pytorch or randpytorch"
raise ValueError(msg)
if ncomp > min(matrix.shape[0], matrix.shape[1]):
msg = '{} PCs cannot be obtained from a matrix with size [{},{}].'
msg += ' Increase the size of the patches or request less PCs'
raise RuntimeError(msg.format(ncomp, matrix.shape[0], matrix.shape[1]))
if mode == 'eigen':
# building the covariance as np.dot(matrix.T,matrix) is slower and takes more memory
C = np.dot(matrix, matrix.T) # covariance matrix
e, EV = linalg.eigh(C) # eigenvalues and eigenvectors
pc = np.dot(EV.T, matrix) # PCs using a compact trick when cov is MM'
V = pc[::-1] # reverse since last eigenvectors are the ones we want
S = np.sqrt(e)[::
-1] # reverse since eigenvalues are in increasing order
if debug:
reconstruction(ncomp, None, S, None)
for i in range(V.shape[1]):
V[:, i] /= S # scaling by the square root of eigenvalues
V = V[:ncomp]
if verbose:
print('Done PCA with numpy linalg eigh functions')
elif mode == 'lapack':
# n_frames is usually smaller than n_pixels. In this setting taking the SVD of M'
# and keeping the left (transposed) SVs is faster than taking the SVD of M (right SVs)
U, S, V = linalg.svd(matrix.T, full_matrices=False)
if debug:
reconstruction(ncomp, U, S, V)
V = V[:ncomp] # we cut projection matrix according to the # of PCs
U = U[:, :ncomp]
S = S[:ncomp]
if verbose:
print('Done SVD/PCA with numpy SVD (LAPACK)')
elif mode == 'arpack':
U, S, V = svds(matrix, k=ncomp)
if debug:
reconstruction(ncomp, U, S, V, -1)
if verbose:
print('Done SVD/PCA with scipy sparse SVD (ARPACK)')
elif mode == 'randsvd':
U, S, V = randomized_svd(matrix,
n_components=ncomp,
n_iter=2,
transpose='auto',
random_state=random_state)
if debug:
reconstruction(ncomp, U, S, V)
if verbose:
print('Done SVD/PCA with randomized SVD')
elif mode == 'cupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
u_gpu, s_gpu, vh_gpu = cupy.linalg.svd(a_gpu,
full_matrices=True,
compute_uv=True)
V = vh_gpu[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if usv:
S = s_gpu[:ncomp]
if to_numpy:
S = cupy.asnumpy(S)
U = u_gpu[:, :ncomp]
if to_numpy:
U = cupy.asnumpy(U)
if verbose:
print('Done SVD/PCA with cupy (GPU)')
elif mode == 'randcupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='cupy')
if to_numpy:
V = cupy.asnumpy(V)
S = cupy.asnumpy(S)
U = cupy.asnumpy(U)
if debug:
reconstruction(ncomp, U, S, V)
if verbose:
print('Done randomized SVD/PCA with cupy (GPU)')
elif mode == 'eigencupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
C = cupy.dot(a_gpu, a_gpu.T) # covariance matrix
e, EV = cupy.linalg.eigh(C) # eigenvalues and eigenvectors
pc = cupy.dot(EV.T, a_gpu) # PCs using a compact trick when cov is MM'
V = pc[::-1] # reverse since last eigenvectors are the ones we want
S = cupy.sqrt(
e)[::-1] # reverse since eigenvalues are in increasing order
if debug:
reconstruction(ncomp, None, S, None)
for i in range(V.shape[1]):
V[:, i] /= S # scaling by the square root of eigenvalues
V = V[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if verbose:
print('Done PCA with cupy eigh function (GPU)')
elif mode == 'pytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32').T))
u_gpu, s_gpu, vh_gpu = torch.svd(a_gpu)
V = vh_gpu[:ncomp]
S = s_gpu[:ncomp]
U = torch.transpose(u_gpu, 0, 1)[:ncomp]
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if verbose:
print('Done SVD/PCA with pytorch (GPU)')
elif mode == 'eigenpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32')))
C = torch.mm(a_gpu, torch.transpose(a_gpu, 0, 1))
e, EV = torch.eig(C, eigenvectors=True)
V = torch.mm(torch.transpose(EV, 0, 1), a_gpu)
S = torch.sqrt(e[:, 0])
if debug:
reconstruction(ncomp, None, S, None)
for i in range(V.shape[1]):
V[:, i] /= S
V = V[:ncomp]
if to_numpy:
V = np.array(V)
if verbose:
print('Done PCA with pytorch eig function')
elif mode == 'randpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='pytorch')
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if debug:
reconstruction(ncomp, U, S, V)
if verbose:
print('Done randomized SVD/PCA with randomized pytorch (GPU)')
else:
raise ValueError('The SVD mode is not available')
if usv:
if mode == 'lapack':
return V.T, S, U.T
elif mode == 'pytorch':
if to_numpy:
return V.T, S, U.T
else:
return torch.transpose(V, 0, 1), S, torch.transpose(U, 0, 1)
else:
return U, S, V
else:
if mode == 'lapack':
return U.T
elif mode == 'pytorch':
return U
else:
return V
def BNM(logits, lw=1):
A = torch.softmax(logits, dim=1)
_, s_tgt, _ = torch.svd(A)
loss_bnm = -torch.mean(s_tgt)
return loss_bnm * lw
def svd_wrapper(matrix, mode, ncomp, verbose, full_output=False,
random_state=None, to_numpy=True):
""" Wrapper for different SVD libraries (CPU and GPU).
Parameters
----------
matrix : numpy ndarray, 2d
2d input matrix.
mode : {'lapack', 'arpack', 'eigen', 'randsvd', 'cupy', 'eigencupy',
'randcupy', 'pytorch', 'eigenpytorch', 'randpytorch'}, str optional
Switch for the SVD method/library to be used.
``lapack``: uses the LAPACK linear algebra library through Numpy
and it is the most conventional way of computing the SVD
(deterministic result computed on CPU).
``arpack``: uses the ARPACK Fortran libraries accessible through
Scipy (computation on CPU).
``eigen``: computes the singular vectors through the
eigendecomposition of the covariance M.M' (computation on CPU).
``randsvd``: uses the randomized_svd algorithm implemented in
Sklearn (computation on CPU).
``cupy``: uses the Cupy library for GPU computation of the SVD as in
the LAPACK version. `
`eigencupy``: offers the same method as with the ``eigen`` option
but on GPU (through Cupy).
``randcupy``: is an adaptation of the randomized_svd algorithm,
where all the computations are done on a GPU (through Cupy). `
`pytorch``: uses the Pytorch library for GPU computation of the SVD.
``eigenpytorch``: offers the same method as with the ``eigen``
option but on GPU (through Pytorch).
``randpytorch``: is an adaptation of the randomized_svd algorithm,
where all the linear algebra computations are done on a GPU
(through Pytorch).
ncomp : int
Number of singular vectors to be obtained. In the cases when the full
SVD is computed (LAPACK, ARPACK, EIGEN, CUPY), the matrix of singular
vectors is truncated.
verbose: bool
If True intermediate information is printed out.
full_output : bool optional
If True the 3 terms of the SVD factorization are returned. If ``mode``
is eigen then only S and V are returned.
random_state : int, RandomState instance or None, optional
If int, random_state is the seed used by the random number generator.
If RandomState instance, random_state is the random number generator.
If None, the random number generator is the RandomState instance used
by np.random. Used for ``randsvd`` mode.
to_numpy : bool, optional
If True (by default) the arrays computed in GPU are transferred from
VRAM and converted to numpy ndarrays.
Returns
-------
V : numpy ndarray
The right singular vectors of the input matrix. If ``full_output`` is
True it returns the left and right singular vectors and the singular
values of the input matrix. If ``mode`` is set to eigen then only S and
V are returned.
References
----------
* For ``lapack`` SVD mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.svd.html
http://www.netlib.org/lapack/
* For ``eigen`` mode see:
https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.linalg.eigh.html
* For ``arpack`` SVD mode see:
https://docs.scipy.org/doc/scipy-0.19.1/reference/generated/scipy.sparse.linalg.svds.html
http://www.caam.rice.edu/software/ARPACK/
* For ``randsvd`` SVD mode see:
https://github.com/scikit-learn/scikit-learn/blob/master/sklearn/utils/extmath.py
Finding structure with randomness: Stochastic algorithms for constructing
approximate matrix decompositions
Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061
* For ``cupy`` SVD mode see:
https://docs-cupy.chainer.org/en/stable/reference/generated/cupy.linalg.svd.html
* For ``eigencupy`` mode see:
https://docs-cupy.chainer.org/en/master/reference/generated/cupy.linalg.eigh.html
* For ``pytorch`` SVD mode see:
http://pytorch.org/docs/master/torch.html#torch.svd
* For ``eigenpytorch`` mode see:
http://pytorch.org/docs/master/torch.html#torch.eig
"""
if matrix.ndim != 2:
raise TypeError('Input matrix is not a 2d array')
if ncomp > min(matrix.shape[0], matrix.shape[1]):
msg = '{} PCs cannot be obtained from a matrix with size [{},{}].'
msg += ' Increase the size of the patches or request less PCs'
raise RuntimeError(msg.format(ncomp, matrix.shape[0], matrix.shape[1]))
if mode == 'eigen':
# building C as np.dot(matrix.T,matrix) is slower and takes more memory
C = np.dot(matrix, matrix.T) # covariance matrix
e, EV = linalg.eigh(C) # EVals and EVs
pc = np.dot(EV.T, matrix) # PCs using a compact trick when cov is MM'
V = pc[::-1] # reverse since we need the last EVs
S = np.sqrt(np.abs(e)) # SVals = sqrt(EVals)
S = S[::-1] # reverse since EVals go in increasing order
for i in range(V.shape[1]):
V[:, i] /= S # scaling EVs by the square root of EVals
V = V[:ncomp]
if verbose:
print('Done PCA with numpy linalg eigh functions')
elif mode == 'lapack':
# n_frames is usually smaller than n_pixels. In this setting taking
# the SVD of M' and keeping the left (transposed) SVs is faster than
# taking the SVD of M (right SVs)
U, S, V = linalg.svd(matrix.T, full_matrices=False)
V = V[:ncomp] # we cut projection matrix according to the # of PCs
U = U[:, :ncomp]
S = S[:ncomp]
if verbose:
print('Done SVD/PCA with numpy SVD (LAPACK)')
elif mode == 'arpack':
U, S, V = svds(matrix, k=ncomp)
if verbose:
print('Done SVD/PCA with scipy sparse SVD (ARPACK)')
elif mode == 'randsvd':
U, S, V = randomized_svd(matrix, n_components=ncomp, n_iter=2,
transpose='auto', random_state=random_state)
if verbose:
print('Done SVD/PCA with randomized SVD')
elif mode == 'cupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
u_gpu, s_gpu, vh_gpu = cupy.linalg.svd(a_gpu, full_matrices=True,
compute_uv=True)
V = vh_gpu[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if full_output:
S = s_gpu[:ncomp]
if to_numpy:
S = cupy.asnumpy(S)
U = u_gpu[:, :ncomp]
if to_numpy:
U = cupy.asnumpy(U)
if verbose:
print('Done SVD/PCA with cupy (GPU)')
elif mode == 'randcupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='cupy')
if to_numpy:
V = cupy.asnumpy(V)
S = cupy.asnumpy(S)
U = cupy.asnumpy(U)
if verbose:
print('Done randomized SVD/PCA with cupy (GPU)')
elif mode == 'eigencupy':
if no_cupy:
raise RuntimeError('Cupy is not installed')
a_gpu = cupy.array(matrix)
a_gpu = cupy.asarray(a_gpu) # move the data to the current device
C = cupy.dot(a_gpu, a_gpu.T) # covariance matrix
e, EV = cupy.linalg.eigh(C) # eigenvalues and eigenvectors
pc = cupy.dot(EV.T, a_gpu) # using a compact trick when cov is MM'
V = pc[::-1] # reverse to get last eigenvectors
S = cupy.sqrt(e)[::-1] # reverse since EVals go in increasing order
for i in range(V.shape[1]):
V[:, i] /= S # scaling by the square root of eigvals
V = V[:ncomp]
if to_numpy:
V = cupy.asnumpy(V)
if verbose:
print('Done PCA with cupy eigh function (GPU)')
elif mode == 'pytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32').T))
u_gpu, s_gpu, vh_gpu = torch.svd(a_gpu)
V = vh_gpu[:ncomp]
S = s_gpu[:ncomp]
U = torch.transpose(u_gpu, 0, 1)[:ncomp]
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if verbose:
print('Done SVD/PCA with pytorch (GPU)')
elif mode == 'eigenpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
a_gpu = torch.Tensor.cuda(torch.from_numpy(matrix.astype('float32')))
C = torch.mm(a_gpu, torch.transpose(a_gpu, 0, 1))
e, EV = torch.eig(C, eigenvectors=True)
V = torch.mm(torch.transpose(EV, 0, 1), a_gpu)
S = torch.sqrt(e[:, 0])
for i in range(V.shape[1]):
V[:, i] /= S
V = V[:ncomp]
if to_numpy:
V = np.array(V)
if verbose:
print('Done PCA with pytorch eig function')
elif mode == 'randpytorch':
if no_torch:
raise RuntimeError('Pytorch is not installed')
U, S, V = randomized_svd_gpu(matrix, ncomp, n_iter=2, lib='pytorch')
if to_numpy:
V = np.array(V)
S = np.array(S)
U = np.array(U)
if verbose:
print('Done randomized SVD/PCA with randomized pytorch (GPU)')
else:
raise ValueError('The SVD `mode` is not recognized')
if full_output:
if mode == 'lapack':
return V.T, S, U.T
elif mode == 'pytorch':
if to_numpy:
return V.T, S, U.T
else:
return torch.transpose(V, 0, 1), S, torch.transpose(U, 0, 1)
elif mode in ('eigen', 'eigencupy', 'eigenpytorch'):
return S, V
else:
return U, S, V
else:
if mode == 'lapack':
return U.T
elif mode == 'pytorch':
return U
else:
return V
path='model/'+str(nn_mass) +'_'+str((100 * float(correct) / total))+'.pt'
torch.save(model_raw, path)
frac=np.linspace(start=0.9, stop=1.1, num = 101)
loss_log=np.zeros([101,101])
#print(model_new.features[0].weight)
for i in range(101):
for j in range(101):
model_new = torch.load(path)
for x in model_new.features:
u,sig,v=torch.svd(x.weight)
sig[0]=sig[0]*frac[i]
sig[1]=sig[1]*frac[j]
tmp=torch.matmul(u,torch.diag(sig))
tmp_v=torch.transpose(v,0,1)
tmp=torch.matmul(tmp,tmp_v)
x.weight.data=tmp
#print(model_new.features[0].weight)
correct = 0
total = 0
predicted_label=torch.zeros(test_num)
image = Variable(test_data)
label = Variable(test_label)
outputs = model_new(image)
def randomized_svd_gpu(M, n_components, n_oversamples=10, n_iter='auto',
transpose='auto', random_state=0, lib='cupy'):
"""Computes a truncated randomized SVD on GPU. Adapted from Sklearn.
Parameters
----------
M : ndarray or sparse matrix
Matrix to decompose
n_components : int
Number of singular values and vectors to extract.
n_oversamples : int (default is 10)
Additional number of random vectors to sample the range of M so as
to ensure proper conditioning. The total number of random vectors
used to find the range of M is n_components + n_oversamples. Smaller
number can improve speed but can negatively impact the quality of
approximation of singular vectors and singular values.
n_iter : int or 'auto' (default is 'auto')
Number of power iterations. It can be used to deal with very noisy
problems. When 'auto', it is set to 4, unless `n_components` is small
(< .1 * min(X.shape)) `n_iter` in which case is set to 7.
This improves precision with few components.
transpose : True, False or 'auto' (default)
Whether the algorithm should be applied to M.T instead of M. The
result should approximately be the same. The 'auto' mode will
trigger the transposition if M.shape[1] > M.shape[0] since this
implementation of randomized SVD tend to be a little faster in that
case.
random_state : int, RandomState instance or None, optional (default=None)
The seed of the pseudo random number generator to use when shuffling
the data. If int, random_state is the seed used by the random number
generator; If RandomState instance, random_state is the random number
generator; If None, the random number generator is the RandomState
instance used by `np.random`.
lib : {'cupy', 'pytorch'}, str optional
Chooses the GPU library to be used.
Notes
-----
This algorithm finds a (usually very good) approximate truncated
singular value decomposition using randomization to speed up the
computations. It is particularly fast on large matrices on which
you wish to extract only a small number of components. In order to
obtain further speed up, `n_iter` can be set <=2 (at the cost of
loss of precision).
References
----------
* Finding structure with randomness: Stochastic algorithms for constructing
approximate matrix decompositions
Halko, et al., 2009 http://arxiv.org/abs/arXiv:0909.4061
* A randomized algorithm for the decomposition of matrices
Per-Gunnar Martinsson, Vladimir Rokhlin and Mark Tygert
* An implementation of a randomized algorithm for principal component
analysis
A. Szlam et al. 2014
"""
random_state = check_random_state(random_state)
n_random = n_components + n_oversamples
n_samples, n_features = M.shape
if n_iter == 'auto':
# Checks if the number of iterations is explicitly specified
n_iter = 7 if n_components < .1 * min(M.shape) else 4
if transpose == 'auto':
transpose = n_samples < n_features
if transpose:
M = M.T # this implementation is a bit faster with smaller shape[1]
if lib == 'cupy':
M = cupy.array(M)
M = cupy.asarray(M)
# Generating normal random vectors with shape: (M.shape[1], n_random)
Q = random_state.normal(size=(M.shape[1], n_random))
Q = cupy.array(Q)
Q = cupy.asarray(Q)
# Perform power iterations with Q to further 'imprint' the top
# singular vectors of M in Q
for i in range(n_iter):
Q = cupy.dot(M, Q)
Q = cupy.dot(M.T, Q)
# Sample the range of M using by linear projection of Q. Extract an orthonormal basis
Q, _ = cupy.linalg.qr(cupy.dot(M, Q), mode='reduced')
# project M to the (k + p) dimensional space using the basis vectors
B = cupy.dot(Q.T, M)
B = cupy.array(B)
Q = cupy.array(Q)
# compute the SVD on the thin matrix: (k + p) wide
Uhat, s, V = cupy.linalg.svd(B, full_matrices=False, compute_uv=True)
del B
U = cupy.dot(Q, Uhat)
if transpose:
# transpose back the results according to the input convention
return V[:n_components, :].T, s[:n_components], U[:,
:n_components].T
else:
return U[:, :n_components], s[:n_components], V[:n_components, :]
elif lib == 'pytorch':
M_gpu = torch.Tensor.cuda(torch.from_numpy(M.astype('float32')))
# Generating normal random vectors with shape: (M.shape[1], n_random)
Q = torch.cuda.FloatTensor(M_gpu.shape[1], n_random).normal_()
# Perform power iterations with Q to further 'imprint' the top
# singular vectors of M in Q
for i in range(n_iter):
Q = torch.mm(M_gpu, Q)
Q = torch.mm(torch.transpose(M_gpu, 0, 1), Q)
# Sample the range of M using by linear projection of Q. Extract an orthonormal basis
Q, _ = torch.qr(torch.mm(M_gpu, Q))
# project M to the (k + p) dimensional space using the basis vectors
B = torch.mm(torch.transpose(Q, 0, 1), M_gpu)
# compute the SVD on the thin matrix: (k + p) wide
Uhat, s, V = torch.svd(B)
del B
U = torch.mm(Q, Uhat)
if transpose:
# transpose back the results according to the input convention
return (torch.transpose(V[:n_components, :], 0, 1),
s[:n_components],
torch.transpose(U[:, :n_components], 0, 1))
else:
return U[:, :n_components], s[:n_components], V[:n_components, :]
def main():
cfg.configure(args)
cfg.print_config()
torch.set_num_threads(args.omp_cores)
torch.manual_seed(args.seed)
model= j1j2.J1J2_C4V_BIPARTITE(j1=args.j1, j2=args.j2, j3=args.j3, \
hz_stag=args.hz_stag, delta_zz=args.delta_zz)
energy_f = model.energy_1x1_lowmem
# energy_f= model.energy_1x1_tiled
# energy_f= model.energy_1x1
# initialize an ipeps
if args.instate != None:
state = read_ipeps_u1(args.instate, vertexToSite=None)
assert len(state.coeffs) == 1, "Not a 1-site ipeps"
state.add_noise(args.instate_noise)
elif args.opt_resume is not None:
if args.bond_dim in [2, 3, 4, 5, 6, 7, 8]:
u1sym_t= tenU1.import_sym_tensors(2,args.bond_dim,"A_1",\
infile=f"u1sym/D{args.bond_dim}_U1_{args.u1_class}.txt",\
dtype=cfg.global_args.torch_dtype, device=cfg.global_args.device)
else:
raise ValueError("Unsupported --bond_dim= " + str(args.bond_dim))
A = torch.zeros(len(u1sym_t),
dtype=cfg.global_args.torch_dtype,
device=cfg.global_args.device)
coeffs = {(0, 0): A}
state = IPEPS_U1SYM(u1sym_t, coeffs)
state.load_checkpoint(args.opt_resume)
elif args.ipeps_init_type == 'RANDOM':
if args.bond_dim in [2, 3, 4, 5, 6, 7, 8]:
u1sym_t= tenU1.import_sym_tensors(2, args.bond_dim, "A_1", \
infile=f"u1sym/D{args.bond_dim}_U1_{args.u1_class}.txt", \
dtype=cfg.global_args.torch_dtype, device=cfg.global_args.device)
else:
raise ValueError("Unsupported --bond_dim= " + str(args.bond_dim))
A = torch.rand(len(u1sym_t),
dtype=cfg.global_args.torch_dtype,
device=cfg.global_args.device)
A = A / torch.max(torch.abs(A))
coeffs = {(0, 0): A}
state = IPEPS_U1SYM(u1sym_t, coeffs)
else:
raise ValueError("Missing trial state: -instate=None and -ipeps_init_type= "\
+str(args.ipeps_init_type)+" is not supported")
print(state)
@torch.no_grad()
def ctmrg_conv_rdm2x1(state, env, history, ctm_args=cfg.ctm_args):
if not history:
history = dict({"log": []})
rdm2x1 = rdm2x1_sl(state, env, force_cpu=ctm_args.conv_check_cpu)
dist = float('inf')
if len(history["log"]) > 1:
dist = torch.dist(rdm2x1, history["rdm"], p=2).item()
# log dist and observables
if args.obs_freq>0 and \
(len(history["log"])%args.obs_freq==0 or
(len(history["log"])-1)%args.obs_freq==0):
e_curr = energy_f(state, env, force_cpu=ctm_args.conv_check_cpu)
obs_values, obs_labels = model.eval_obs(state, env, force_cpu=True)
print(
", ".join([f"{len(history['log'])}", f"{dist}", f"{e_curr}"] +
[f"{v}" for v in obs_values]))
else:
print(f"{len(history['log'])}, {dist}")
# update history
history["rdm"] = rdm2x1
history["log"].append(dist)
converged = dist < ctm_args.ctm_conv_tol
if converged or len(history['log']) >= ctm_args.ctm_max_iter:
log.info({
"history_length": len(history['log']),
"history": history['log'],
"final_multiplets": compute_multiplets(env)
})
return converged, history
return False, history
ctm_env_init = ENV_C4V(args.chi, state)
init_env(state, ctm_env_init)
e_curr0 = energy_f(state, ctm_env_init, force_cpu=True)
obs_values0, obs_labels = model.eval_obs(state,
ctm_env_init,
force_cpu=True)
print(", ".join(["epoch", "energy"] + obs_labels))
print(", ".join([f"{-1}", f"{e_curr0}"] + [f"{v}" for v in obs_values0]))
ctm_env_init, *ctm_log = ctmrg_c4v.run(state, ctm_env_init, \
conv_check=ctmrg_conv_rdm2x1)
e_curr0 = energy_f(state, ctm_env_init, force_cpu=True)
obs_values0, obs_labels = model.eval_obs(state,
ctm_env_init,
force_cpu=True)
history, t_ctm, t_obs = ctm_log
print("\n")
print(", ".join(["epoch", "energy"] + obs_labels))
print("FINAL " + ", ".join([f"{e_curr0}"] + [f"{v}" for v in obs_values0]))
print(f"TIMINGS ctm: {t_ctm} conv_check: {t_obs}")
# ----- additional observables ---------------------------------------------
corrSS = model.eval_corrf_SS(state,
ctm_env_init,
args.corrf_r,
canonical=args.corrf_canonical)
print("\n\nSS r " + " ".join([label for label in corrSS.keys()]) +
f" canonical {args.corrf_canonical}")
for i in range(args.corrf_r):
print(f"{i} " +
" ".join([f"{corrSS[label][i]}" for label in corrSS.keys()]))
corrDD = model.eval_corrf_DD_H(state, ctm_env_init, args.corrf_r)
print("\n\nDD r " + " ".join([label for label in corrDD.keys()]))
for i in range(args.corrf_r):
print(f"{i} " +
" ".join([f"{corrDD[label][i]}" for label in corrDD.keys()]))
if args.corrf_dd_v:
corrDD_V = model.eval_corrf_DD_V(state, ctm_env_init, args.corrf_r)
print("\n\nDD_v r " + " ".join([label for label in corrDD_V.keys()]))
for i in range(args.corrf_r):
print(f"{i} " + " ".join(
[f"{corrDD_V[label][i]}" for label in corrDD_V.keys()]))
# environment diagnostics
print("\n\nspectrum(C)")
u, s, v = torch.svd(ctm_env_init.C[ctm_env_init.keyC], compute_uv=False)
for i in range(args.chi):
print(f"{i} {s[i]}")
# transfer operator spectrum 1-site-width channel
print("\n\nspectrum(T)")
l = transferops_c4v.get_Top_spec_c4v(args.top_n, state, ctm_env_init)
for i in range(l.size()[0]):
print(f"{i} {l[i,0]} {l[i,1]}")
# transfer operator spectrum 2-site-width channel
if args.top2:
print("\n\nspectrum(T2)")
l = transferops_c4v.get_Top2_spec_c4v(args.top_n, state, ctm_env_init)
for i in range(l.size()[0]):
print(f"{i} {l[i,0]} {l[i,1]}")