def test_tokenize(self):
expected = [
Token("Name", "foo"),
Token("Operator", "&&"),
Token("Name", "bar")
]
self.assertEqual(self.tokenizer.tokenize_string("foo && bar"),
expected)
def transform_raw_token(self, raw_token: RawToken) -> Token:
token_type, token_value = raw_token
token_type = str(token_type).replace("Token.", "")
if not self.include_values:
token_value = None
return Token(token_type, token_value)
def test_as_dict(self):
self.assertEqual(Token("foo").as_dict(), {"type": "foo"})
self.assertEqual(Token("foo", "bar").as_dict(), {"type": "foo", "value": "bar"})
def test_repr(self):
self.assertEqual(str(Token("foo")), "Token(type='foo')")
self.assertEqual(str(Token("foo", "bar")), "Token(type='foo', value='bar')")
def test_hash(self):
self.assertEqual(hash(Token("foo")), hash(Token("foo")))
self.assertEqual(hash(Token("foo", "bar")), hash(Token("foo", "bar")))
self.assertNotEqual(hash(Token("foo", "bar")), hash(Token("foo")))
def test_eq(self):
self.assertEqual(Token("foo"), Token("foo"))
self.assertEqual(Token("foo", "bar"), Token("foo", "bar"))
self.assertNotEqual(Token("foo", "bar"), Token("foo"))
def test_transform_and_token(self):
self.assertEqual(self._get_token("&& bar"), Token("Operator", "&&"))
def test_transform_normal_token(self):
self.assertEqual(self._get_token("this.foo"), Token("Keyword", "this"))