def test_sequence_learn(self):
sp = StreamPredictor()
first_words_list = ['that', 'is', 'good']
sp.train(first_words_list)
first_next_words, first_probabilities = sp.pop_manager.get('that').get_next_smallest_distribution()
self.assertEqual(['is'], first_next_words)
self.assertEqual([1.0], first_probabilities)
second_words_list = ['that', 'isnt', 'good']
sp.train(second_words_list)
second_next_words, second_probabilities = sp.pop_manager.get('that').get_next_smallest_distribution()
self.assertEqual({'is', 'isnt'}, set(second_next_words))
self.assertListEqual([0.5, 0.5], list(second_probabilities))
def test_sequence_learn(self):
sp = StreamPredictor()
first_words_list = ['that', 'is', 'good']
sp.train(first_words_list)
first_next_words, first_probabilities = sp.pop_manager.get(
'that').get_next_smallest_distribution()
self.assertEqual(['is'], first_next_words)
self.assertEqual([1.0], first_probabilities)
second_words_list = ['that', 'isnt', 'good']
sp.train(second_words_list)
second_next_words, second_probabilities = sp.pop_manager.get(
'that').get_next_smallest_distribution()
self.assertEqual({'is', 'isnt'}, set(second_next_words))
self.assertListEqual([0.5, 0.5], list(second_probabilities))
import sys
import matplotlib.pyplot as plt
sys.path.insert(0, '../streampredictor/')
sys.path.insert(0, '../')
from streampredictor.stream_predictor import StreamPredictor
from streampredictor import data_fetcher
input_text_file = '../Data/ptb.test.txt'
max_input_length = 10 ** 4
words = data_fetcher.get_clean_words_from_file(input_text_file, max_input_length)
sp = StreamPredictor()
test_length = -1000
sp.train(words[:test_length])
sp.file_manager.save_tsv('../PatternStore/ptb.tsv')
perplexity_list = sp.calculate_perplexity(words=words[test_length:], verbose=True)
plt.plot(perplexity_list)
plt.xlabel('Time')
plt.ylabel('Perplexity')
plt.title('Perplexity during training')
plt.show()
import sys
sys.path.insert(0, '../streampredictor/')
sys.path.insert(0, '../')
from streampredictor.stream_predictor import StreamPredictor
sp = StreamPredictor()
sp.occasional_step = 100
input_text = 'hello how are you hello who are you'
input_words = input_text.split(' ') * 1000
print(input_words[:20])
sp.train(input_words, verbose=True)
print('\n\nThe generated words are ')
print(sp.generate(20))
print('\nEnd of generated words\n')
sp.file_manager.save_tsv('../PatternStore/simple_generate.tsv')
print(sp.pop_manager.pattern_collection)
def get_sample():
sample = StreamPredictor()
sample.train(training_words)
return sample
def get_sample():
sample = StreamPredictor()
sample.train(training_words)
return sample