def test_tokenize_record(self):
normalizer = TokenizerNormalizer(attr_index=1)
self.assertEqual(['a1', [u"J'", u'aime', u'les', u'frites', u'!',]],
normalizer.normalize(['a1', u"J'aime les frites !"]))
def test_tokenize(self):
normalizer = TokenizerNormalizer()
self.assertEqual([u"J'", u'aime', u'les', u'frites', u'!',],
normalizer.normalize(u"J'aime les frites !"))