def _prepare_test_data(ngram_order): return ( Vocabulary(["a", "b", "c", "d", "z", "<s>", "</s>"], unk_cutoff=1), [ list(padded_everygrams(ngram_order, sent)) for sent in (list("abcd"), list("egadbe")) ], )
def _prepare_test_data(ngram_order): return ( Vocabulary(["a", "b", "c", "d", "z", "<s>", "</s>"], unk_cutoff=1), [ list(padded_everygrams(ngram_order, sent)) for sent in (list("abcd"), list("egadbe")) ], )
def test_generate_cycle(self): # Add a cycle to the model: bd -> b, db -> d more_training_text = [list(padded_everygrams(self.model.order, list("bdbdbd")))] self.model.fit(more_training_text) # Test that we can escape the cycle self.assertEqual( self.model.generate(7, text_seed=("b", "d"), random_seed=5), ["b", "d", "b", "d", "b", "d", "</s>"], )
def test_generate_cycle(self): # Add a cycle to the model: bd -> b, db -> d more_training_text = [list(padded_everygrams(self.model.order, list("bdbdbd")))] self.model.fit(more_training_text) # Test that we can escape the cycle self.assertEqual( self.model.generate(7, text_seed=("b", "d"), random_seed=5), ["b", "d", "b", "d", "b", "d", "</s>"], )
def test_generate_cycle(mle_trigram_model): # Add a cycle to the model: bd -> b, db -> d more_training_text = [padded_everygrams(mle_trigram_model.order, list("bdbdbd"))] mle_trigram_model.fit(more_training_text) # Test that we can escape the cycle assert mle_trigram_model.generate(7, text_seed=("b", "d"), random_seed=5) == [ "b", "d", "b", "d", "b", "d", "</s>", ]
def trigram_training_data(training_data): return [list(padded_everygrams(3, sent)) for sent in training_data]