def test_tokenize_record(self): normalizer = TokenizerNormalizer(attr_index=1) self.assertEqual(['a1', [u"J'", u'aime', u'les', u'frites', u'!',]], normalizer.normalize(['a1', u"J'aime les frites !"]))
def test_tokenize(self): normalizer = TokenizerNormalizer() self.assertEqual([u"J'", u'aime', u'les', u'frites', u'!',], normalizer.normalize(u"J'aime les frites !"))