def test_common_sense_perplexity_2(self):
assert score("I decided that I wanted ice cream.", stride=1) < score("I decided that I wanted shelf.", stride=1)
def test_common_sense_perplexity_3(self):
assert score("The grass was green.", stride=1) < score("The grass was red.", stride=1)
def test_common_sense_perplexity_1(self):
assert score("The exams were due at noon.", stride=1) < score("The exams were due at exam.", stride=1)
def test_perplexity_calculation_5(self):
if model_id == 'gpt2':
assert math.isclose(score("the the the the the the", stride=1),
math.exp((4.6442 + 6.6947 + 3.8328 + 1.4247 + 1.1237) / 5), rel_tol=0.001)
def test_perplexity_calculation_4(self):
if model_id == 'gpt2':
assert math.isclose(score("Do they know what happened to John?", stride=1),
math.exp((5.4927 + 2.9635 + 1.7828 + 4.1989 + 1.2399 + 6.6663 + 2.0147) / 7),
rel_tol=0.0001)
def test_perplexity_calculation_3(self):
if model_id == 'gpt2':
assert math.isclose(score("She was a kind person.", stride=1),
math.exp((3.6758 + 2.7864 + 6.0170 + 2.7715 + 1.8178) / 5), rel_tol=0.0001)
def test_perplexity_calculation_2(self):
if model_id == 'gpt2':
assert math.isclose(score("He went to the library", stride=1),
math.exp((6.5366 + 1.3047 + 1.5648 + 4.9886) / 4), rel_tol=0.0001)