def test_perplexity_with_all_skips(self): output_probs = np.array([[0.2, 0.3, 0.5], [1.0, 0.0, 0.0]]) output_tokens = np.array([0, 0]) sequence_likelihood = np.log(output_probs[np.arange(len(output_probs)), output_tokens]).sum() skip_token_id = 0 with self.assertRaises(MetricsException): _calculate_mean_perplexity(np.array([output_tokens]), np.array([sequence_likelihood]), skip_token_id)
def test_perplexity(self): output_probs = np.array([[0.2, 0.3, 0.5], [1.0, 0.0, 0.0]]) output_tokens = np.array([0, 0]) sequence_likelihood = np.log(output_probs[np.arange(len(output_probs)), output_tokens]).sum() perp = _calculate_mean_perplexity(np.array([output_tokens]), np.array([sequence_likelihood]), 2) perp_exp = np.exp(-0.5 * np.log(0.2)) self.assertAlmostEqual(perp, perp_exp, delta=_DELTA)
def test_perplexity_with_skip_token_exp(self): output_probs = np.array([[0.2, 0.3, 0.5], [1.0, 0.0, 0.0]]) output_tokens = np.array([2, 1]) skip_token_id = 1 mask = output_tokens != skip_token_id sequence_likelihood = np.log(output_probs[np.arange(len(output_probs[mask])), output_tokens[mask]]).sum() perp_exp = np.exp(-np.log(0.5)) perp = _calculate_mean_perplexity(np.array([output_tokens]), np.array([sequence_likelihood]), skip_token_id) self.assertAlmostEqual(perp, perp_exp, delta=_DELTA)