def test_escape_token(self): token = u"abc_\\4" alphabet = set("abc_\\u;") escaped_token = tokenizer._escape_token(token, alphabet) self.assertEqual("abc\\u\\\\\\52;_", escaped_token)
def test_escape_token(self): token = u"abc_\\4" alphabet = set("abc_\\u;") escaped_token = tokenizer._escape_token(token, alphabet) self.assertEqual("abc\\u\\\\\\52;_", escaped_token)