def test_transform_two_empty_documents(self):
vectorizer = VoikkoAttributeVectorizer(['NUMBER'])
X = vectorizer.transform(['', '.'])
self.assertIsInstance(X, csr_matrix)
self.assertEqual((2, 3), X.shape)
data = X.toarray()
self.assertEqual(0, np.count_nonzero(data))
def test_transform_ambiguity(self):
vectorizer = VoikkoAttributeVectorizer(['MOOD'])
X = vectorizer.transform(['tulen'])
self.assertEqual((1, 10), X.shape)
data = X.toarray()
self.assertEqual(1, np.count_nonzero(data))
self.assertEqual(0.5, data[0][8])
def test_build_tokenizer(self):
vectorizer = VoikkoAttributeVectorizer([])
tokenizer = vectorizer.build_tokenizer()
tokens = tokenizer('Kissa ei ole koira. Ei todellakaan ole.')
self.assertEqual(
['Kissa', 'ei', 'ole', 'koira', 'Ei', 'todellakaan', 'ole'],
tokens)
def test_get_feature_names(self):
vectorizer = VoikkoAttributeVectorizer(['NUMBER', 'PERSON'])
names = vectorizer.get_feature_names()
expected = [
'unknown', 'NUMBER_plural', 'NUMBER_singular', 'PERSON_1',
'PERSON_2', 'PERSON_3', 'PERSON_4'
]
self.assertEqual(expected, names)
def test_transform(self):
vectorizer = VoikkoAttributeVectorizer(['NUMBER'])
X = vectorizer.transform(['Kissa ja jaf koira.'])
self.assertIsInstance(X, csr_matrix)
self.assertEqual((1, 3), X.shape)
data = X.toarray()
self.assertEqual(0.25, data[0][0]) # unknown
self.assertEqual(0, data[0][1]) # plural
self.assertEqual(0.5, data[0][2]) # singular
def test_fit_transform(self):
# should do the same as fit
vectorizer = VoikkoAttributeVectorizer(['NUMBER'])
X = vectorizer.fit_transform(['Kissa ja jaf koira.'])
self.assertIsInstance(X, csr_matrix)
def test_fit(self):
vectorizer = VoikkoAttributeVectorizer(['NUMBER'])
# fit should not do anything, just return self
self.assertIs(vectorizer, vectorizer.fit(['Kissa ei ole koira.']))
def test_get_feature_names_non_categorial_feature_raises(self):
self.assertRaises(ValueError,
lambda: VoikkoAttributeVectorizer(['BASEFORM']))
def test_get_feature_names_unknown_feature_raises(self):
self.assertRaises(ValueError,
lambda: VoikkoAttributeVectorizer(['KISSA']))
def test_init_and_terminate(self):
v = VoikkoAttributeVectorizer([])
v.terminate()