def __init__(self, channels):
super().__init__()
self.channels = channels
shape = (channels, channels)
rotation_mat = np.linalg.qr(
np.random.normal(size=shape))[0].astype("float32")
# w_p: a permutation matrix
# w_l: a lower triangular matrix with ones on the diagonal
# w_u: an upper triangular matrix with zeros on the diagonal,
w_p, w_l, w_u = scipy.linalg.lu(rotation_mat)
s = cf.diagonal(w_u)
u_mask = np.triu(np.ones_like(w_u), k=1)
l_mask = np.tril(np.ones_like(w_u), k=-1)
l_diag = np.eye(w_l.shape[0])
w_u = w_u * u_mask
with self.init_scope():
self.w_u = chainer.Parameter(initializer=w_u, shape=w_u.shape)
self.w_l = chainer.Parameter(initializer=w_l, shape=w_l.shape)
self.s = chainer.Parameter(initializer=s, shape=s.shape)
self.add_persistent("w_p", w_p)
self.add_persistent("u_mask", u_mask)
self.add_persistent("l_mask", l_mask)
self.add_persistent("l_diag", l_diag)
# W^(-1) is not a learnable parameter
inv_rotation_mat = np.linalg.inv(self.W)
self.inverse_conv = L.Convolution2D(channels,
channels,
ksize=1,
stride=1,
pad=0,
nobias=True,
initialW=inv_rotation_mat)
def _pairwise_distances_l2(embeddings):
"""Compute the 2D matrix of distances between all the embeddings.
Distance is defined by L2 norm: distance(x, y) := ||x - y||^2
Args:
embeddings: Variable with shape=(batch_size, embed_dim)
Returns:
pairwise_distances: Variable with shape=(batch_size, batch_size)
"""
# Multiply 0.5 to embeddings because constrain distance in [0.0, 1.0].
embeddings = 0.5 * embeddings
# Get the dot product between all embeddings
# shape (batch_size, batch_size)
dot_product = F.matmul(embeddings, embeddings, transa=False, transb=True)
# Get squared L2 norm for each embedding. We can just take the diagonal of `dot_product`.
# This also provides more numerical stability (the diagonal of the result will be exactly 0).
# shape (batch_size,)
squared_norm = F.diagonal(dot_product)
# Compute the pairwise distance matrix as we have:
# ||a - b||^2 = ||a||^2 - 2 <a, b> + ||b||^2
# shape (batch_size, batch_size)
distances = F.expand_dims(
squared_norm, axis=0) - 2.0 * dot_product + F.expand_dims(squared_norm,
axis=1)
# Because of computation errors, some distances might be negative so we put everything >= 0.0
distances = F.clip(distances, 0.0, 1.0)
return distances
def _kdeparts(self, input_obs, input_ins):
""" Multivariate Kernel Density Estimation (KDE) with Gaussian kernels on the given random variables.
INPUT:
input_obs - Variable of input observation random variables to estimate density
input_ins - Variable of input data instance to calculate the probability value
OUTPUT:
const - Constant term in the Gaussian KDE expression
energy - Expressions in the exponential to calculate Gaussian KDE (energy wrt. every obs. point)
"""
[n, d] = input_obs.shape
# Compute Kernel Bandwidth Matrix based on Silverman's Rule of Thumb
silverman_factor = np.power(n * (d + 2.0) / 4.0, -1. / (d + 4))
input_centered = input_obs - F.mean(input_obs, axis=0, keepdims=True)
data_covariance = F.matmul(F.transpose(input_centered), input_centered) / n
kernel_bw = F.diagonal(data_covariance) * (silverman_factor ** 2) * np.eye(d, d)
const = 1 / (n * ((2 * np.pi) ** (d/2)) * F.sqrt(F.det(kernel_bw)))
# Compute energy expressions in the exponent for every observation point
diff = input_obs - input_ins
energy = -0.5 * F.diagonal(F.matmul(F.matmul(diff, F.inv(kernel_bw)), F.transpose(diff)))
return const, energy
def regularize_diag_off_diag_dip(cov_mu, lambda_od, lambda_d):
"""Compute on and off diagonal regularizers for DIP-VAE models.
Penalize deviations of covariance_matrix from the identity matrix. Uses
different weights for the deviations of the diagonal and off diagonal entries.
Args:
cov_mu : [num_latent, num_latent]
to regularize.
lambda_od: Weight of penalty for off diagonal elements.
lambda_d: Weight of penalty for diagonal elements.
Returns:
dip_regularizer: Regularized deviation from diagonal of covariance_matrix.
"""
xp = cov_mu.xp
cov_mu_diag = F.diagonal(cov_mu)
cov_mu_off_diag = cov_mu - cov_mu_diag * xp.eye(cov_mu.shape[0])
dip_regularizer = lambda_od * F.sum(cov_mu_off_diag ** 2) \
+ lambda_d * F.sum((cov_mu_diag - 1) ** 2)
return dip_regularizer
def calcAttentionLoss(self, A):
if self.I is None:
self.setIdentityMatrix(A)
# rewrite to use comprehension like
# F.sum([... for a in A])
'''
loss = 0
for a in A:
# |AA^T|_F^2 = |B|_F^2 = tr(BB^T) = sum(BB^T*I)
b = F.matmul(a.T, a)-self.I
loss += F.sum(F.diagonal(F.matmul(b,b.T)))
'''
losses = []
for a in A:
# |AA^T|_F^2 = |B|_F^2 = tr(BB^T) = sum(BB^T*I)
b = F.matmul(a.T, a) - self.I
losses.append(F.sum(F.diagonal(F.matmul(b, b.T))))
return F.average(F.vstack(losses))
def f(x):
x = functions.diagonal(x, *self.args)
return x * x
def check_backward(self, x_data, y_grad):
gradient_check.check_backward(
lambda x: functions.diagonal(x, *self.args),
x_data,
y_grad,
dtype=numpy.float64)
def check_forward(self, x_data):
x = chainer.Variable(x_data)
y = functions.diagonal(x, *self.args)
testing.assert_allclose(y.data, self.y_expected)
def f(x):
return functions.diagonal(x, *self.args)
def W(self):
kernel = self.w_p @ (self.w_l * self.l_mask + self.l_diag) @ (
self.w_u * self.u_mask + cf.diagonal(self.s))
return cf.reshape(kernel, kernel.shape + (1, 1))
def forward(self, inputs, device):
x, = inputs
return functions.diagonal(x, *self.args),
def f(x):
return functions.diagonal(x, *self.args)
def check_backward(self, x_data, y_grad):
gradient_check.check_backward(
lambda x: functions.diagonal(x, *self.args),
x_data, y_grad, dtype=numpy.float64)
def check_forward(self, x_data):
x = chainer.Variable(x_data)
y = functions.diagonal(x, *self.args)
testing.assert_allclose(y.data, self.y_expected)
def f(x):
x = functions.diagonal(x, *self.args)
return x * x
def forward(self, inputs, device):
x, = inputs
return functions.diagonal(x, *self.args),