def test_tokenizer_lower(self): p = Preprocessor(lower=True, stem=False) tokens = p.tokenizer('This is IRLib') self.assertEqual(tokens,['this','is','irlib'])
def test_tokenizer_lower(self): p = Preprocessor(lower=True, stem=False) tokens = p.tokenizer('This is IRLib') self.assertEqual(tokens, ['this', 'is', 'irlib'])