def solve_with_SVD_v2(self, adj, lamb=1):
# NS version of GraRep
# first construct A^k with normalization
deg_inv = 1 / (torch_sparse.sum(adj, dim=1).view(-1, 1))
A_tilde = torch_sparse.mul(adj, deg_inv)
A = [A_tilde]
if self.k >= 2:
for i in range(1, self.k):
A.append(A[i - 1].matmul(A_tilde))
# then obtain W via low-rank SVD
W = []
for k, Ak in enumerate(A, 1):
print('solving SVD with k={k}'.format(k=k))
tau_k = Ak.sum(dim=0).view(1, -1)
# remove the conditional probability term in the objective function
# to make it a pure negative sampling model
temp = Ak.storage.value()
temp[temp > 0] = 1
temp[temp <= 0] = 0
Ak.storage._value = temp
Xk = torch_sparse.mul(Ak, self.num_embeddings / (tau_k * lamb))
temp = torch.log(Xk.storage.value() + 1e-15)
temp[temp < 0] = 0
Xk.storage._value = temp
Xk = Xk.to_scipy('coo')
u, s, vt = svds(Xk, k=self.embedding_dim) # torch.svd_lowrank does not work due to a bug
Wk = torch.tensor(u * s ** 0.5)
W.append(Wk)
return torch.cat(W, dim=1)
def solve_with_SVD_v1(self, adj, lamb=1):
# the original NCE version of GraRep
# first construct A^k with normalization
deg_inv = 1 / (torch_sparse.sum(adj, dim=1).view(-1, 1))
A_tilde = torch_sparse.mul(adj, deg_inv)
A = [A_tilde]
if self.k >= 2:
for i in range(1, self.k):
A.append(A[i - 1].matmul(A_tilde))
# then obtain W via low-rank SVD
W = []
C = []
for k, Ak in enumerate(A, 1):
print('solving SVD with k={k}'.format(k=k))
tau_k = Ak.sum(dim=0).view(1, -1)
Xk = torch_sparse.mul(Ak, self.num_embeddings / (tau_k * lamb))
temp = torch.log(Xk.storage.value() + 1e-15)
temp[temp < 0] = 0
Xk.storage._value = temp
Xk = Xk.to_scipy('coo')
u, s, vt = svds(Xk, k=self.embedding_dim) # torch.svd_lowrank does not work due to a bug
Wk = torch.tensor(u * s ** 0.5)
Ck = torch.tensor(vt.T * s ** 0.5)
W.append(Wk)
C.append(Ck)
W = torch.cat(W, dim=1)
C = torch.cat(C, dim=1)
H = torch.cat((W, C), dim=1)
return W
def solve_with_SVD_v3(self, adj, lamb=1):
# the version that directly adopts the objective function from the paper:
# Neural Word Embedding as Implicit Matrix Factorization
# first construct A^k without normalization
A = [adj]
if self.k >= 2:
for i in range(1, self.k):
A.append(A[i - 1].matmul(adj))
# then obtain W via low-rank SVD
W = []
for k, Ak in enumerate(A, 1):
print('solving SVD with k={k}'.format(k=k))
num_c = Ak.sum(dim=0).view(1, -1)
num_w = Ak.sum(dim=1).view(-1, 1)
D = Ak.sum()
Xk = torch_sparse.mul(Ak, 1 / num_w)
Xk = torch_sparse.mul(Xk, D / (num_c * lamb))
temp = torch.log(Xk.storage.value() + 1e-15)
temp[temp < 0] = 0
Xk.storage._value = temp
Xk = Xk.to_scipy('coo')
u, s, vt = svds(Xk, k=self.embedding_dim) # torch.svd_lowrank does not work due to a bug
Wk = torch.tensor(u * s ** 0.5)
W.append(Wk)
return torch.cat(W, dim=1)
def gcn_norm(adj_t):
adj_t = torch_sparse.fill_diag(adj_t, 1) # add self-loop
deg = torch_sparse.sum(adj_t, dim=1).pow_(-0.5) # compute normalized degree matrix
deg.masked_fill_(deg == float('inf'), 0.) # for numerical stability
adj_t = torch_sparse.mul(adj_t, deg.view(-1, 1)) # row-wise mul
adj_t = torch_sparse.mul(adj_t, deg.view(1, -1)) # col-wise mul
return adj_t
def norm(self, edge_index):
adj = edge_index
deg = sum(adj, dim=1)
deg_inv = deg.pow_(-1)
deg_inv.masked_fill_(deg_inv == float('inf'), 0.)
adj = mul(adj, deg_inv.view((-1, 1)))
return adj
def gcn_norm(edge_index,
edge_weight=None,
num_nodes=None,
improved=False,
add_self_loops=True,
flow="source_to_target",
dtype=None):
fill_value = 2. if improved else 1.
if isinstance(edge_index, SparseTensor):
assert flow in ["source_to_target"]
adj_t = edge_index
if not adj_t.has_value():
adj_t = adj_t.fill_value(1., dtype=dtype)
if add_self_loops:
adj_t = fill_diag(adj_t, fill_value)
deg = sparsesum(adj_t, dim=1)
deg_inv_sqrt = deg.pow_(-0.5)
deg_inv_sqrt.masked_fill_(deg_inv_sqrt == float('inf'), 0.)
adj_t = mul(adj_t, deg_inv_sqrt.view(-1, 1))
adj_t = mul(adj_t, deg_inv_sqrt.view(1, -1))
return adj_t
else:
assert flow in ["source_to_target", "target_to_source"]
num_nodes = maybe_num_nodes(edge_index, num_nodes)
if edge_weight is None:
edge_weight = torch.ones((edge_index.size(1), ),
dtype=dtype,
device=edge_index.device)
if add_self_loops:
edge_index, tmp_edge_weight = add_remaining_self_loops(
edge_index, edge_weight, fill_value, num_nodes)
assert tmp_edge_weight is not None
edge_weight = tmp_edge_weight
row, col = edge_index[0], edge_index[1]
idx = col if flow == "source_to_target" else row
deg = scatter_add(edge_weight, idx, dim=0, dim_size=num_nodes)
deg_inv_sqrt = deg.pow_(-0.5)
deg_inv_sqrt.masked_fill_(deg_inv_sqrt == float('inf'), 0)
return edge_index, deg_inv_sqrt[row] * edge_weight * deg_inv_sqrt[col]
def _sogcn_norm(edge_index,
edge_weight=None,
num_nodes=None,
improved=False,
add_self_loops=True,
dtype=None):
fill_value = 2. if improved else 1.
if isinstance(edge_index, torch_sparse.SparseTensor):
adj_t = edge_index
if not adj_t.has_value():
adj_t = adj_t.fill_value(1.)
if add_self_loops:
adj_t = torch_sparse.fill_diag(adj_t, fill_value)
deg = sum(adj_t, dim=1)
deg_inv_sqrt = deg.pow_(-0.5)
deg_inv_sqrt.masked_fill_(deg_inv_sqrt == float('inf'), 0.)
adj_t = torch_sparse.mul(adj_t, deg_inv_sqrt.view(-1, 1))
adj_t = torch_sparse.mul(adj_t, deg_inv_sqrt.view(1, -1))
return adj_t
else:
num_nodes = maybe_num_nodes(edge_index, num_nodes)
if edge_weight is None:
edge_weight = torch.ones((edge_index.shape[1], ),
dtype=dtype,
device=edge_index.device)
if add_self_loops:
edge_index, tmp_edge_weight = add_remaining_self_loops(
edge_index, edge_weight, fill_value, num_nodes)
assert tmp_edge_weight is not None
edge_weight = tmp_edge_weight
row, col = edge_index[0], edge_index[1]
deg = torch_scatter.scatter_add(edge_weight,
col,
dim=0,
dim_size=num_nodes)
deg_inv_sqrt = deg.pow_(-0.5)
deg_inv_sqrt.masked_fill_(deg_inv_sqrt == float('inf'), 0)
return edge_index, deg_inv_sqrt[row] * edge_weight * deg_inv_sqrt[col]
