def setUp(self):
self.corpus = Corpus.from_file("deerwester")
self.pp_list = [preprocess.LowercaseTransformer(),
preprocess.WordPunctTokenizer(),
preprocess.SnowballStemmer(),
preprocess.NGrams(),
tag.AveragedPerceptronTagger()]
def test_call_with_tokens(self):
corpus = preprocess.WordPunctTokenizer()(self.corpus)
corpus = self.transformer(corpus)
tokens = ['namuH', 'enihcam', 'ecafretni', 'rof', 'bal', 'cba',
'retupmoc', 'snoitacilppa']
self.assertEqual(corpus.tokens[0], tokens)
self.assertTrue(corpus.has_tokens())
text = 'Human machine interface for lab abc computer applications'
self.assertEqual(corpus.documents[0], text)
self.assertEqual(len(corpus.used_preprocessor.preprocessors), 2)
def set_corpus(self, data=None):
self.corpus = data
# create preprocessed corpus upon setting data to avoid preprocessing
# at each method run
pp_list = [
preprocess.LowercaseTransformer(),
preprocess.WordPunctTokenizer()
]
self.pp_corpus = PreprocessorList(pp_list)(self.corpus)
self.commit()
def test_pos_filter(self):
pos_filter = preprocess.PosTagFilter("NN")
pp_list = [
preprocess.WordPunctTokenizer(),
tag.AveragedPerceptronTagger()
]
corpus = self.corpus
for pp in pp_list:
corpus = pp(corpus)
filtered = pos_filter(corpus)
self.assertTrue(len(filtered.pos_tags))
self.assertEqual(len(filtered.pos_tags[0]), 5)
self.assertEqual(len(filtered.tokens[0]), 5)
def set_corpus(self, data=None):
self.corpus = data
self.pp_corpus = None
if self.corpus is not None:
if not self.corpus.has_tokens():
# create preprocessed corpus upon setting data to avoid
# preprocessing at each method run
pp_list = [
preprocess.LowercaseTransformer(),
preprocess.WordPunctTokenizer()
]
self.pp_corpus = PreprocessorList(pp_list)(self.corpus)
else:
self.pp_corpus = self.corpus
self.commit.now()
def test_filter_pos_tags(self):
pp_list = [
preprocess.LowercaseTransformer(),
preprocess.WordPunctTokenizer(),
tag.AveragedPerceptronTagger(),
preprocess.StopwordsFilter()
]
corpus = self.corpus
with corpus.unlocked():
corpus.metas[0, 0] = "This is the most beautiful day in the world"
for pp in pp_list:
corpus = pp(corpus)
self.assertEqual(len(corpus.tokens), len(corpus.pos_tags))
self.assertEqual(len(corpus.tokens[0]), len(corpus.pos_tags[0]))
self.assertEqual(corpus.tokens[0], ["beautiful", "day", "world"])
self.assertEqual(corpus.pos_tags[0], ["JJ", "NN", "NN"])
def test_preprocessed(self):
widget = self.create_widget(OWSentimentAnalysis)
corpus = self.corpus.copy()
pp_list = [
preprocess.LowercaseTransformer(),
preprocess.WordPunctTokenizer()
]
for pp in pp_list:
corpus = pp(corpus)
self.send_signal(widget.Inputs.corpus, corpus)
self.assertTrue(widget.pp_corpus)
widget.liu_hu.click()
simulate.combobox_activate_item(widget.liu_lang, "English")
self.assertTrue(widget.pp_corpus)
self.send_signal(widget.Inputs.corpus, None)
self.assertIsNone(widget.pp_corpus)
def test_pickle_corpus(self):
"""
Corpus must be picklable (for save data widget)
gh-590
"""
c = Corpus.from_file('book-excerpts')
# it must also work with preprocessed corpus
self.pp_list = [
preprocess.LowercaseTransformer(),
preprocess.WordPunctTokenizer(),
preprocess.SnowballStemmer(),
preprocess.FrequencyFilter(),
preprocess.StopwordsFilter()
]
for pp in self.pp_list:
c = pp(c)
pickle.dumps(c)