def test_sampled_softmax_has_greater_loss_in_train_mode(self): sampled_softmax = SampledSoftmaxLoss(num_words=10000, embedding_dim=12, num_samples=10) # sequence_length, embedding_dim embedding = torch.rand(100, 12) targets = torch.randint(0, 1000, (100,)).long() sampled_softmax.train() train_loss = sampled_softmax(embedding, targets).item() sampled_softmax.eval() eval_loss = sampled_softmax(embedding, targets).item() assert eval_loss > train_loss
def test_sampled_equals_unsampled_during_eval(self): sampled_softmax = SampledSoftmaxLoss(num_words=10000, embedding_dim=12, num_samples=40) unsampled_softmax = _SoftmaxLoss(num_words=10000, embedding_dim=12) sampled_softmax.eval() unsampled_softmax.eval() # set weights equal, use transpose because opposite shapes sampled_softmax.softmax_w.data = unsampled_softmax.softmax_w.t() sampled_softmax.softmax_b.data = unsampled_softmax.softmax_b # sequence_length, embedding_dim embedding = torch.rand(100, 12) targets = torch.randint(0, 1000, (100,)).long() full_loss = unsampled_softmax(embedding, targets).item() sampled_loss = sampled_softmax(embedding, targets).item() # Should be really close np.testing.assert_almost_equal(sampled_loss, full_loss)