def test_split_string_to_tokens(self): text = "test? testing 123." tokens = tokenizer._split_string_to_tokens(text) self.assertEqual(["test", "? ", "testing", "123", "."], tokens)
def test_split_string_to_tokens(self): text = "test? testing 123." tokens = tokenizer._split_string_to_tokens(text) self.assertEqual(["test", "? ", "testing", "123", "."], tokens)