def test_decode_text_ideal(self):
corpus = ('i', 'have', 'a', 'cat', '<END>', 'his', 'name', 'is',
'bruno', '<END>', 'i', 'have', 'a', 'dog', 'too', '<END>',
'his', 'name', 'is', 'rex', '<END>', 'her', 'name', 'is',
'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(3, encoded)
context = (
storage.get_id('name'),
storage.get_id('is'),
)
print('Я ТЕСТ', context)
end = storage.get_id('<END>')
generator = LikelihoodBasedTextGenerator(storage, trie)
to_decode = generator.generate_text(context, 2)
self.assertEqual(to_decode[-1], end)
expected = ('Name is rex', 'Her name is rex')
actual = decode_text(storage, to_decode)
self.assertEqual(expected, actual)
def test_decode_text_incorrect_storage(self):
corpus = ('i', 'have', 'a', 'cat', '<END>', 'his', 'name', 'is',
'bruno', '<END>', 'i', 'have', 'a', 'dog', 'too', '<END>',
'his', 'name', 'is', 'rex', '<END>', 'her', 'name', 'is',
'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(3, encoded)
context = (
storage.get_id('name'),
storage.get_id('is'),
)
generator = LikelihoodBasedTextGenerator(storage, trie)
to_decode = generator.generate_text(context, 2)
bad_inputs = [(), [], 123, None, NGramTrie]
for bad_storage in bad_inputs:
self.assertRaises(ValueError, decode_text, bad_storage, to_decode)
def test_decode_text_ideal_conditions(self):
corpus = ('i', 'have', 'a', 'cat', '<END>', 'his', 'name', 'is',
'bruno', '<END>', 'i', 'have', 'a', 'dog', 'too', '<END>',
'his', 'name', 'is', 'rex', '<END>', 'her', 'name', 'is',
'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(3, encoded)
context = (
storage.get_id('name'),
storage.get_id('is'),
)
generator = LikelihoodBasedTextGenerator(storage, trie)
to_decode = generator.generate_text(context, 2)
actual = decode_text(storage, to_decode)
for sentence in actual:
self.assertTrue('<END>' not in sentence)
self.assertTrue(sentence[0].isupper())
self.assertTrue(sentence[-1].isalpha())
def test_end(self):
"""
Checks that after decoding no end in result
"""
corpus = ('i', 'have', 'a', 'cat', '<END>', 'his', 'name', 'is',
'bruno', '<END>', 'i', 'have', 'a', 'cat', '<END>', 'his',
'name', 'is', 'rex', '<END>', 'her', 'name', 'is', 'cat',
'<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(3, encoded)
context = (
storage.get_id('a'),
storage.get_id('cat'),
)
generator = LikelihoodBasedTextGenerator(storage, trie)
to_decode = generator.generate_text(context, 1)
actual = decode_text(storage, to_decode)
expected = ('A cat', )
self.assertEqual(expected, actual)
def realize_likelihood_generator(text):
likelihood_storage = WordStorage()
likelihood_storage.update(text)
context = (likelihood_storage.get_id('i'),
likelihood_storage.get_id('shall'),)
model = load_model('lab_4/likelihood_model.json')
generator = LikelihoodBasedTextGenerator(model.word_storage, model.n_gram_trie)
likelihood_text_generated = generator.generate_text(context, 3)
return decode_text(likelihood_storage, likelihood_text_generated)
def test_decode_text_upper_first_letter(self):
'''
Tests that number all the letters except
first one in a sentence are in a lower case
'''
corpus = ('first', 'sentence', 'here', '<END>', 'second', 'sentence',
'here', '<END>', 'third', 'sentence', 'here', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded_text = encode_text(storage, corpus)
trie = NGramTrie(3, encoded_text)
context = (storage.get_id('first'), storage.get_id('sentence'))
likelihood_generator = LikelihoodBasedTextGenerator(storage, trie)
generated_encoded_text = likelihood_generator.generate_text(context, 1)
decoded_text = decode_text(storage, generated_encoded_text)
self.assertFalse(decoded_text[0][1:].isupper())
def test_likelihood_generator_instance_creation(self):
"""
Checks that class creates correct instance
"""
word_storage = WordStorage()
ngram = NGramTrie(2, ())
generator = LikelihoodBasedTextGenerator(word_storage, ngram)
self.assertEqual(generator._word_storage, word_storage)
self.assertEqual(generator._n_gram_trie, ngram)
def test_generate_next_word_larger_context(self):
corpus = ('i', 'have', 'a', 'cat', '<END>', 'his', 'name', 'is',
'bruno', '<END>', 'i', 'have', 'a', 'dog', 'too', '<END>',
'his', 'name', 'is', 'rex', '<END>', 'her', 'name', 'is',
'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(4, encoded)
expected_word = storage.get_id('bruno')
context = (storage.get_id('his'), storage.get_id('name'),
storage.get_id('is'))
generator = LikelihoodBasedTextGenerator(storage, trie)
actual = generator._generate_next_word(context)
self.assertEqual(expected_word, actual)
def test_float_result(self):
"""
Checks that returned result is float
"""
corpus = ('i', 'have', 'a', 'cat', '<END>', 'his', 'name', 'is',
'bruno', '<END>', 'i', 'have', 'a', 'dog', 'too', '<END>',
'his', 'name', 'is', 'rex', '<END>', 'her', 'name', 'is',
'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(2, encoded)
context = (storage.get_id('i'), )
word = storage.get_id('have')
generator = LikelihoodBasedTextGenerator(storage, trie)
actual = generator._calculate_maximum_likelihood(word, context)
self.assertEqual(float, type(actual))
def test_calculate_likelihood_no_such_context(self):
corpus = ('i', 'have', 'a', 'cat', '<END>',
'his', 'name', 'is', 'bruno', '<END>',
'i', 'have', 'a', 'dog', 'too', '<END>',
'his', 'name', 'is', 'rex', '<END>',
'her', 'name', 'is', 'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(3, encoded)
word = storage.get_id('dog')
context = (storage.get_id('<END>'),
storage.get_id('<END>'),)
generator = LikelihoodBasedTextGenerator(storage, trie)
expected = 0.0
actual = generator._calculate_maximum_likelihood(word, context)
self.assertEqual(expected, actual)
def test_generate_next_word_no_context(self):
"""
Checks that next word generates even if context isn't found
"""
corpus = ('he', 'likes', 'a', 'cat', 'but', 'he', 'does', 'not',
'like', 'parrots', '<END>', 'he', 'says', 'that', 'his',
'name', 'is', 'bruno', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(3, encoded)
context = (
storage.get_id('cat'),
storage.get_id('is'),
)
generator = LikelihoodBasedTextGenerator(storage, trie)
expected_most_frequent = storage.get_id('he')
actual = generator._generate_next_word(context)
self.assertEqual(expected_most_frequent, actual)
def test_generate_next_word_incorrect_context(self):
corpus = ('i', 'have', 'a', 'cat', '<END>', 'his', 'name', 'is',
'bruno', '<END>', 'i', 'have', 'a', 'dog', 'too', '<END>',
'his', 'name', 'is', 'rex', '<END>', 'her', 'name', 'is',
'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(3, encoded)
bad_inputs = [[], {}, (2000, ), None, 9, 9.34, True]
generator = LikelihoodBasedTextGenerator(storage, trie)
for bad_context in bad_inputs:
self.assertRaises(ValueError, generator._generate_next_word,
bad_context)
def test_calculate_likelihood_incorrect_context(self):
corpus = ('i', 'have', 'a', 'cat', '<END>',
'his', 'name', 'is', 'bruno', '<END>',
'i', 'have', 'a', 'dog', 'too', '<END>',
'his', 'name', 'is', 'rex', '<END>',
'her', 'name', 'is', 'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(2, encoded)
bad_inputs = [[], {}, (2000, 1000, ), None, 9, 9.34, True] # (2000, 1000, ) -> context for three gram
word = storage.get_id('dog')
generator = LikelihoodBasedTextGenerator(storage, trie)
for bad_context in bad_inputs:
self.assertRaises(ValueError,
generator._calculate_maximum_likelihood,
word, bad_context)
def test_calculate_likelihood_incorrect_word(self):
corpus = ('i', 'have', 'a', 'cat', '<END>',
'his', 'name', 'is', 'bruno', '<END>',
'i', 'have', 'a', 'dog', 'too', '<END>',
'his', 'name', 'is', 'rex', '<END>',
'her', 'name', 'is', 'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(2, encoded)
bad_inputs = [(), [], None, 123]
context = (storage.get_id('have'),
storage.get_id('a'),)
generator = LikelihoodBasedTextGenerator(storage, trie)
for bad_word in bad_inputs:
self.assertRaises(ValueError,
generator._calculate_maximum_likelihood,
bad_word, context)
if __name__ == '__main__':
corpus = ('i', 'have', 'a', 'cat', '<END>', 'his', 'name', 'is', 'bruno',
'<END>', 'i', 'have', 'a', 'dog', 'too', '<END>', 'his', 'name',
'is', 'rex', '<END>', 'her', 'name', 'is', 'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = ngrams.NGramTrie(2, encoded)
context = (storage.get_id('i'), )
generator = NGramTextGenerator(storage, trie)
actual = generator.generate_text(context, 5)
actual = decode_text(storage, actual)
print(actual)
generator = LikelihoodBasedTextGenerator(storage, trie)
actual = generator.generate_text(context, 5)
actual = decode_text(storage, actual)
print(actual)
two = ngrams.NGramTrie(2, encoded)
trie = ngrams.NGramTrie(3, encoded)
context = (
storage.get_id('name'),
storage.get_id('is'),
)
generator = BackOffGenerator(storage, trie, two)
actual = generator.generate_text(context, 5)
"""
from lab_4.main import LikelihoodBasedTextGenerator, encode_text, WordStorage, decode_text
from lab_4.ngrams.ngram_trie import NGramTrie
if __name__ == '__main__':
corpus = ('i', 'have', 'a', 'cat', '<END>', 'his', 'name', 'is', 'bruno',
'<END>', 'i', 'have', 'a', 'dog', 'too', '<END>', 'his', 'name',
'is', 'rex', '<END>', 'her', 'name', 'is', 'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(3, encoded)
context = (
storage.get_id('name'),
storage.get_id('is'),
)
end = storage.get_id('<END>')
generator = LikelihoodBasedTextGenerator(storage, trie)
to_decode = generator.generate_text(context, 2)
EXPECTED = ('Name is rex', 'Her name is rex')
RESULT = decode_text(storage, to_decode)
assert RESULT == EXPECTED, 'Encoding not working'
from lab_4.main import tokenize_by_sentence
from lab_4.main import WordStorage, LikelihoodBasedTextGenerator
from lab_4.main import encode_text, decode_text
from lab_4.main import NGramTextGenerator
if __name__ == '__main__':
TEXT = 'I have a cat. His name is Bruno. I have a dog too. His name is Rex. Her name is Rex too.'
tokenized_text = tokenize_by_sentence(TEXT)
word_storage = WordStorage()
word_storage.update(tokenized_text)
encoded = encode_text(word_storage, tokenized_text)
trie = NGramTrie(3, encoded)
context = (
word_storage.get_id('name'),
word_storage.get_id('is'),
)
generator = NGramTextGenerator(word_storage, trie)
generated_text = generator.generate_text(context, 2)
gen_likelihood = LikelihoodBasedTextGenerator(word_storage, trie)
gen_text = gen_likelihood.generate_text(context, 2)
decoded_text = decode_text(word_storage, gen_text)
RESULT = decoded_text
assert RESULT == ('Name is rex', 'Her name is rex'), "Not working"
The school library where Oleg studies is good. It is a large clean room.
There are four big windows in it. The walls are light blue. There are a lot of shelves full of books.
You can find books on literature, physics, history, chemistry, geography, biology and other subjects.
There are books in English, too. On the walls you can see pictures of some great writers and poets.
On the table near the window you can always see beautiful spring and autumn flowers.
Oleg likes to go to the library. He can always find there something new, something he needs.
'''
corpus = tokenize_by_sentence(TEXT)
print(f'TOKENIZE_BY_SENTENCE RESULT: {corpus}\n')
word_storage = WordStorage()
word_storage.update(corpus)
encoded_text = encode_text(word_storage, corpus)
print(f'ENCODE_TEXT RESULT: {encoded_text}\n')
trie = NGramTrie(4, encoded_text)
context = (word_storage.get_id('the'), word_storage.get_id('walls'),
word_storage.get_id('are'))
likelihood_generator = LikelihoodBasedTextGenerator(word_storage, trie)
generated_text = likelihood_generator.generate_text(context, 3)
print(f'ENCODED_GENERATED_TEXT: {generated_text}\n')
decoded_text = decode_text(word_storage, generated_text)
print(f'DECODED_GENERATED_TEXT: {decoded_text}')
RESULT = decoded_text
assert RESULT == (
'The walls are light blue', 'There are books in english too',
'On the walls you can see pictures of some great writers and poets'
), 'Generator not working'
"Lab 4 implementation starter"
from lab_4.main import WordStorage, encode_text, LikelihoodBasedTextGenerator, decode_text
from lab_4.ngrams.ngram_trie import NGramTrie
if __name__ == '__main__':
corpus = ('i', 'have', 'a', 'cat', '<END>', 'his', 'name', 'is', 'bruno',
'<END>', 'i', 'have', 'a', 'dog', 'too', '<END>', 'his', 'name',
'is', 'rex', '<END>', 'her', 'name', 'is', 'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded_text = encode_text(storage, corpus)
n_gram_trie = NGramTrie(3, encoded_text)
context = (
storage.get_id('i'),
storage.get_id('have'),
)
generator = LikelihoodBasedTextGenerator(storage, n_gram_trie)
generated_text = generator.generate_text(context, 3)
RESULT = decode_text(storage, generated_text)
print(RESULT)
# DO NOT REMOVE NEXT LINE - KEEP IT INTENTIONALLY LAST
assert RESULT == ('I have a cat', 'His name is rex',
'Her name is rex'), 'Encoding not working'
'colourful', 'pets', 'too', '<END>', 'they', 'have', 'beautiful',
'dogs', '<END>', 'i', 'havent', 'a', 'cat', '<END>', 'i',
'havent', 'a', 'cat', 'too', '<END>', 'we', 'havent', 'a', 'cat',
'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(4, encoded)
context = (storage.get_id('i'), storage.get_id('have'),
storage.get_id('a'))
generator_likelihood = LikelihoodBasedTextGenerator(storage, trie)
generated_text = generator_likelihood.generate_text(context, 3)
decoded_gen_text = decode_text(storage, generated_text)
print('Likelihood generator generates sentences:')
print(*decoded_gen_text, sep='. ', end='.\n')
two = NGramTrie(2, encoded)
trie = NGramTrie(3, encoded)
context = (
storage.get_id('i'),
storage.get_id('have'),
)
generator_backoff = BackOffGenerator(storage, trie, two)
