def _prepare_test_data(ngram_order):
return (
Vocabulary(["a", "b", "c", "d", "z", "<s>", "</s>"], unk_cutoff=1),
[
list(padded_everygrams(ngram_order, sent))
for sent in (list("abcd"), list("egadbe"))
],
)
def _prepare_test_data(ngram_order):
return (
Vocabulary(["a", "b", "c", "d", "z", "<s>", "</s>"], unk_cutoff=1),
[
list(padded_everygrams(ngram_order, sent))
for sent in (list("abcd"), list("egadbe"))
],
)
def test_generate_cycle(self):
# Add a cycle to the model: bd -> b, db -> d
more_training_text = [list(padded_everygrams(self.model.order, list("bdbdbd")))]
self.model.fit(more_training_text)
# Test that we can escape the cycle
self.assertEqual(
self.model.generate(7, text_seed=("b", "d"), random_seed=5),
["b", "d", "b", "d", "b", "d", "</s>"],
)
def test_generate_cycle(self):
# Add a cycle to the model: bd -> b, db -> d
more_training_text = [list(padded_everygrams(self.model.order, list("bdbdbd")))]
self.model.fit(more_training_text)
# Test that we can escape the cycle
self.assertEqual(
self.model.generate(7, text_seed=("b", "d"), random_seed=5),
["b", "d", "b", "d", "b", "d", "</s>"],
)
def test_generate_cycle(mle_trigram_model):
# Add a cycle to the model: bd -> b, db -> d
more_training_text = [padded_everygrams(mle_trigram_model.order, list("bdbdbd"))]
mle_trigram_model.fit(more_training_text)
# Test that we can escape the cycle
assert mle_trigram_model.generate(7, text_seed=("b", "d"), random_seed=5) == [
"b",
"d",
"b",
"d",
"b",
"d",
"</s>",
]
def trigram_training_data(training_data):
return [list(padded_everygrams(3, sent)) for sent in training_data]
