def test_join_tokens_to_string(self): tokens = ["test", "? ", "testing", "123", "."] s = tokenizer._join_tokens_to_string(tokens) self.assertEqual("test? testing 123.", s)
def test_join_tokens_to_string(self): tokens = ["test", "? ", "testing", "123", "."] s = tokenizer._join_tokens_to_string(tokens) self.assertEqual("test? testing 123.", s)