def compute_kernel(self,
x1: torch.tensor,
x2: torch.tensor) -> torch.tensor:
# Convert the tensors into row and column vectors
D = x1.size(1)
N = x1.size(0)
x1 = x1.unsqueeze(-2) # Make it into a column tensor
x2 = x2.unsqueeze(-3) # Make it into a row tensor
"""
Usually the below lines are not required, especially in our case,
but this is useful when x1 and x2 have different sizes
along the 0th dimension.
"""
x1 = x1.expand(N, N, D)
x2 = x2.expand(N, N, D)
if self.kernel_type == 'rbf':
result = self.compute_rbf(x1, x2)
elif self.kernel_type == 'imq':
result = self.compute_inv_mult_quad(x1, x2)
else:
raise ValueError('Undefined kernel type.')
return result
def _sample_noise(self, x: torch.tensor, num: int,
batch_size) -> np.ndarray:
""" Sample the base classifier's prediction under noisy corruptions of the input x.
:param x: the input [channel x width x height]
:param num: number of samples to collect
:param batch_size:
:return: an ndarray[int] of length num_classes containing the per-class counts
"""
with torch.no_grad():
x = x.unsqueeze(0)
prior_params = self.cvae.prior(x)
latent_dim = prior_params[0].size(1)
counts = 0
for _ in range(ceil(num / batch_size)):
this_batch_size = min(batch_size, num)
num -= this_batch_size
# batch = x.repeat((this_batch_size, 1, 1, 1))
noise = torch.randn(this_batch_size, latent_dim,
device='cuda') * self.sigma
# predictions = self.base_classifier(batch + noise).argmax(1)
# Feed noise into CVAE instead of adding to input
z = self.cvae.reparameterize(prior_params, eps=noise)
batch = self.cvae.decode(
x.expand((this_batch_size, -1, -1, -1)), z)
predictions = self.base_classifier(batch).max(1)[1]
if predictions.ndim == 3:
counts += self._count_tensor(predictions, self.num_classes)
else:
counts += self._count_arr(predictions, self.num_classes)
return counts
def step(self, word: torch.tensor, last_hidden_state: tuple,
domains_embedding: torch.tensor, domains_mask: torch.tensor):
'''
Perform a step of LSTM Cell.
@param word of shape (batch_size, word_embedding_dim): w_t in paper
@param last_hidden_state, tuple of shape ((batch_size, hidden_size), (batch_size, hidden_size)): hidden_state and cell at time t-1, h_{t-1} and c_{t-1} in paper
@param domains_embedding of shape (batch_size, max_domains_num, domain_embedding_dim): input question domains embedding, u in paper
@param domains_mask of shape (batch_size, max_domains_num): mask of domains (mask of src or padding)
@return hidden_state contains (hidden_hidden of shape (batch_size, hidden_size), hidden_cell)
'''
batch_size = word.size(0)
word = torch.unsqueeze(word, dim=1)
expand_word = word.expand(batch_size, self.max_domains_num,
word.size(2))
if last_hidden_state is None:
expanded_hidden_state = torch.zeros(
(batch_size, self.max_domains_num, self.hidden_size),
dtype=torch.float).to(self.device)
else:
expanded_hidden_state = torch.unsqueeze(
last_hidden_state[0],
dim=1).expand(last_hidden_state[0].size(0),
self.max_domains_num,
last_hidden_state[0].size(1))
attention_mat = torch.cat(
[domains_embedding, expand_word, expanded_hidden_state], dim=2)
w_proj = torch.matmul(attention_mat, self.W_ac)
v_proj = torch.matmul(torch.tanh(w_proj), self.V_ac).squeeze(2)
v_proj.data.masked_fill_(domains_mask.bool(), float('-inf'))
# shape (batch_size, max_domains_num)
alpha = nn.functional.softmax(v_proj, dim=1)
u_t = torch.bmm(alpha.unsqueeze(1), domains_embedding).squeeze(
1) # shape (batch_size, domain_embedding_dim)
x = torch.cat([word.squeeze(1), u_t], dim=1)
hidden_state = self.attention_LSTM.forward(x, last_hidden_state)
return hidden_state
