def test_lemmatizer_record(self): normalizer = LemmatizerNormalizer(FRENCH_LEMMAS, attr_index=1) self.assertEqual(['a1', u'sacré rubert'], normalizer.normalize(['a1', u'sacré rubert'])) self.assertEqual(['a1', u'je aimer le frite'], normalizer.normalize(['a1', u"J'aime les frites !"])) self.assertEqual(['a1', u'je aimer le frite'], normalizer.normalize(['a1', u", J'aime les frites"]))
def test_lemmatizer(self): normalizer = LemmatizerNormalizer(FRENCH_LEMMAS) self.assertEqual(normalizer.normalize(u'sacré rubert'), u'sacré rubert') self.assertEqual(normalizer.normalize(u"J'aime les frites !"), u'je aimer le frite') self.assertEqual(normalizer.normalize(u", J'aime les frites"), u'je aimer le frite')