def test_generate_text_large_context(self):
"""
should generate simple case with three sentences out of small corpus
"""
corpus = ('i', 'have', 'a', 'cat', '<END>', 'his', 'name', 'is',
'bruno', '<END>', 'i', 'have', 'a', 'dog', 'too', '<END>',
'his', 'name', 'is', 'rex', '<END>', 'her', 'name', 'is',
'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = NGramTrie(5, encoded)
generator = NGramTextGenerator(storage, trie)
context = (
storage.get_id('i'),
storage.get_id('have'),
storage.get_id('a'),
storage.get_id('bruno'),
)
end = storage.get_id('<END>')
actual = generator.generate_text(context, 3)
self.assertEqual(actual.count(end), 3)
def test_load_model_takes_less_time(self):
with open('lab_1/data.txt', 'r', encoding='utf-8') as big_file:
big_data = big_file.read()
tokenized_data = tokenize_by_sentence(big_data)
storage = WordStorage()
storage.update(tokenized_data)
context = (
storage.get_id('despite'),
storage.get_id('the'),
)
start_time_generate = time()
encoded = encode_text(storage, tokenized_data)
trie = NGramTrie(3, encoded)
generator = NGramTextGenerator(storage, trie)
generated_text = generator.generate_text(context, 3)
end_time_generate = time() - start_time_generate
save_model(generator, 'model_training')
start_time_saved = time()
loaded_model = load_model('model_training')
new_result = loaded_model.generate_text(context, 3)
end_time_saved = time() - start_time_saved
self.assertGreater(end_time_generate, end_time_saved)
self.assertEqual(generated_text, new_result)
def realize_n_gram_text_generator(text):
n_gram_storage = WordStorage()
n_gram_storage.update(text)
n_gram_context = (n_gram_storage.get_id('my'), n_gram_storage.get_id('dear'))
n_gram_encoded = encode_text(n_gram_storage, text)
n_gram_trie = NGramTrie(3, n_gram_encoded)
n_gram_generator = NGramTextGenerator(n_gram_storage, n_gram_trie)
n_gram_text_generated = n_gram_generator.generate_text(n_gram_context, 3)
return decode_text(n_gram_storage, n_gram_text_generated)
from lab_4.main import tokenize_by_sentence, WordStorage, encode_text, NGramTextGenerator
from lab_4.ngrams.ngram_trie import NGramTrie
if __name__ == '__main__':
TEXT = 'I like flowers.\nMy mom likes flowers too.\nHer favourite flower is rose.\nMy favourite flower is rose too.'
corpus = tokenize_by_sentence(TEXT)
word_storage = WordStorage()
word_storage.update(corpus)
encoded_corpus = encode_text(word_storage, corpus)
ngrams = NGramTrie(2, encoded_corpus)
text_generator = NGramTextGenerator(word_storage, ngrams)
gen_text = text_generator.generate_text((1, ), 2)
end = word_storage.get_id('<END>')
actual = gen_text.count(end)
RESULT = 2
print(actual)
assert RESULT == actual, 'not working'
from lab_4.main import NGramTextGenerator
from lab_4.main import LikelihoodBasedTextGenerator
from lab_4.main import BackOffGenerator
if __name__ == '__main__':
corpus = ('i', 'have', 'a', 'cat', '<END>', 'his', 'name', 'is', 'bruno',
'<END>', 'i', 'have', 'a', 'dog', 'too', '<END>', 'his', 'name',
'is', 'rex', '<END>', 'her', 'name', 'is', 'rex', 'too', '<END>')
storage = WordStorage()
storage.update(corpus)
encoded = encode_text(storage, corpus)
trie = ngrams.NGramTrie(2, encoded)
context = (storage.get_id('i'), )
generator = NGramTextGenerator(storage, trie)
actual = generator.generate_text(context, 5)
actual = decode_text(storage, actual)
print(actual)
generator = LikelihoodBasedTextGenerator(storage, trie)
actual = generator.generate_text(context, 5)
actual = decode_text(storage, actual)
print(actual)
two = ngrams.NGramTrie(2, encoded)
trie = ngrams.NGramTrie(3, encoded)
context = (
storage.get_id('name'),
storage.get_id('is'),
)
"""
Lab 4 implementation starter
"""
from lab_4.main import tokenize_by_sentence, WordStorage, encode_text, NGramTextGenerator
from lab_4.ngrams.ngram_trie import NGramTrie
if __name__ == '__main__':
text = 'I have a dog.\nHis name is Will'
tokenize_text = tokenize_by_sentence(text)
print(tokenize_text)
storage = WordStorage()
storage.update(tokenize_text)
print(storage)
encode = encode_text(storage, tokenize_text)
print(encode)
n_gram_trie = NGramTrie(2, encode)
print(n_gram_trie)
generator = NGramTextGenerator(storage, n_gram_trie)
context = (storage.get_id('a'), )
print(context)
RESULT = generator.generate_text(context, 3)
print(RESULT)
# DO NOT REMOVE NEXT LINE - KEEP IT INTENTIONALLY LAST
assert RESULT, 'Not working'
from lab_4.main import tokenize_by_sentence
from lab_4.main import WordStorage, LikelihoodBasedTextGenerator
from lab_4.main import encode_text, decode_text
from lab_4.main import NGramTextGenerator
if __name__ == '__main__':
TEXT = 'I have a cat. His name is Bruno. I have a dog too. His name is Rex. Her name is Rex too.'
tokenized_text = tokenize_by_sentence(TEXT)
word_storage = WordStorage()
word_storage.update(tokenized_text)
encoded = encode_text(word_storage, tokenized_text)
trie = NGramTrie(3, encoded)
context = (
word_storage.get_id('name'),
word_storage.get_id('is'),
)
generator = NGramTextGenerator(word_storage, trie)
generated_text = generator.generate_text(context, 2)
gen_likelihood = LikelihoodBasedTextGenerator(word_storage, trie)
gen_text = gen_likelihood.generate_text(context, 2)
decoded_text = decode_text(word_storage, gen_text)
RESULT = decoded_text
assert RESULT == ('Name is rex', 'Her name is rex'), "Not working"
"""
Text generator implementation starter
"""
from lab_4.main import WordStorage, NGramTrie, NGramTextGenerator
from lab_4.main import tokenize_by_sentence, encode_text
if __name__ == '__main__':
# here goes your function calls
first_text = open('lab_3/Frank_Baum.txt', encoding="utf-8")
first_text_tokenized = tokenize_by_sentence(first_text.read())
word_storage = WordStorage()
word_storage.update(first_text_tokenized)
encoded = encode_text(word_storage, first_text_tokenized)
n_gram_trie = NGramTrie(2, encoded)
generator = NGramTextGenerator(word_storage, n_gram_trie)
RESULT = generator.generate_text(encoded[16:17], 3)
#print(RESULT)
assert RESULT, "Not working"
if __name__ == '__main__':
text = tokenize_by_sentence(
"""Hi everyone! Nice to meet you again. What are you doing in my laboratory work?
You are very nice person, do you know it? To be honest, I can't stand programming.
But it doesn't depend on you! It's my personal problem and I don't know how to
solve it... It doesn't matter right now""")
word_storage = WordStorage()
word_storage.update(text)
encoded_text = encode_text(word_storage, text)
n_gram_trie = NGramTrie(3, encoded_text)
generator_of_text = NGramTextGenerator(word_storage, n_gram_trie)
context = word_storage.get_id('on'), word_storage.get_id('you')
formed_ids = generator_of_text.generate_text(context, 1)
formed_text = []
for ids in formed_ids:
word = word_storage.get_word(ids)
if word != '<END>':
formed_text.append(word)
RESULT = ' '.join(formed_text)
print(RESULT)
assert RESULT == 'on you', ''
from lab_4.main import encode_text
from lab_4.main import NGramTextGenerator
if __name__ == '__main__':
text = 'I have a cat. His name is Bruno. I have a dog too. ' \
'His name is Rex. Her name is Rex too'
corpus = tokenize_by_sentence(text)
word_storage = WordStorage()
word_storage.update(corpus)
encoded_text = encode_text(word_storage, corpus)
n_gram_trie = NGramTrie(2, encoded_text)
n_gram_text_generator = NGramTextGenerator(word_storage, n_gram_trie)
context = (word_storage.get_id('i'), word_storage.get_id('have'))
text_generated = n_gram_text_generator.generate_text(context, 2)
output_text = []
for word_id in text_generated:
word = word_storage.get_word(word_id)
if word != '<END>':
output_text.append(word)
RESULT = ' '.join(output_text)
print(RESULT)
assert RESULT == 'i have a cat name is rex', 'Something went wrong :('
import lab_4
from lab_4.main import WordStorage, tokenize_by_sentence, encode_text, NGramTextGenerator
from lab_4.ngrams.ngram_trie import NGramTrie
if __name__ == '__main__':
text = 'I have a cat.\nHer name is Mila'
tok_text = tokenize_by_sentence(text)
storage = WordStorage()
storage.update(tok_text)
print(storage)
encoded_text = encode_text(storage, tok_text)
print(encoded_text)
n_gram_trie = NGramTrie(2, encoded_text)
print(n_gram_trie)
gen = NGramTextGenerator(storage, n_gram_trie)
context = (storage.get_id('a'), )
print(context)
RESULT = gen.generate_text(context, 2)
print(RESULT)
assert RESULT, 'Not working'
from lab_4.ngrams.ngram_trie import NGramTrie
from lab_4.main import tokenize_by_sentence
from lab_4.main import WordStorage
from lab_4.main import encode_text
from lab_4.main import NGramTextGenerator
if __name__ == '__main__':
text = "This is a dog. It likes running. This is a cat. It likes sleeping. Everyone likes sleeping too."
text_in_tokens = tokenize_by_sentence(text)
word_storage = WordStorage()
word_storage.update(text_in_tokens)
encoded_text = encode_text(word_storage, text_in_tokens)
n_gram_trie = NGramTrie(2, encoded_text)
context = (word_storage.get_id('likes'),)
text_generator = NGramTextGenerator(word_storage, n_gram_trie)
RESULT = text_generator.generate_text(context, 4)
print(RESULT)
assert RESULT, "Someting went worng.."