def test_read_number(self):
token_stream = TokenStream(InputStream('123='))
result = token_stream._read_number()
self.assertEqual(Token('num', 123.0), result)
token_stream = TokenStream(InputStream('123.3.='))
result = token_stream._read_number()
self.assertEqual(Token('num', 123.3), result)
def test_read_string(self):
token_stream = TokenStream(InputStream('"ab"'))
result = token_stream._read_string()
self.assertEqual(Token('str', 'ab'), result)
token_stream = TokenStream(InputStream('"ab\\c"'))
result = token_stream._read_string()
self.assertEqual(Token('str', 'abc'), result)
token_stream = TokenStream(InputStream('"abc'))
with self.assertRaises(Exception):
token_stream._read_string()
def test_read_ident(self):
token_stream = TokenStream(InputStream('a=1'))
result = token_stream._read_identifier()
self.assertEqual(Token('var', 'a=1'), result)
token_stream = TokenStream(InputStream('a = 1'))
result = token_stream._read_identifier()
self.assertEqual(Token('var', 'a'), result)
token_stream = TokenStream(InputStream('let(a = 1'))
result = token_stream._read_identifier()
self.assertEqual(Token('kw', 'let'), result)
token_stream = TokenStream(InputStream('js "aaa"'))
result = token_stream._read_identifier()
self.assertEqual(Token('kw', 'js'), result)
def test_read_next(self):
token_stream = TokenStream(
InputStream(' # comment\n123 abc "nba" let a=2 >= js;'))
self.assertEqual(token_stream._read_next(), Token('num', 123.0))
self.assertEqual(token_stream._read_next(), Token('var', 'abc'))
self.assertEqual(token_stream._read_next(), Token('str', 'nba'))
self.assertEqual(token_stream._read_next(), Token('kw', 'let'))
self.assertEqual(token_stream._read_next(), Token('var', 'a=2'))
self.assertEqual(token_stream._read_next(), Token('op', '>='))
self.assertEqual(token_stream._read_next(), Token('kw', 'js'))
self.assertEqual(token_stream._read_next(), Token('punc', ';'))
self.assertEqual(token_stream._read_next(), Token('null', 'null'))
token_stream = TokenStream(InputStream('\x08'))
with self.assertRaises(Exception):
token_stream._read_next()
token_stream = TokenStream(InputStream('λ (n) 1'))
self.assertEqual(token_stream._read_next(), Token("kw", 'λ'))
def _is_op(self, char: str) -> bool:
token = self._token_stream.peek()
return token == Token('op', char)
def _is_punc(self, char: str) -> bool:
token = self._token_stream.peek()
return token == Token('punc', char)
def test_peek_and_next(self):
token_stream = TokenStream(
InputStream(' # comment\n123 abc let a=2 >=;'))
self.assertEqual(token_stream.peek(), Token('num', 123.0))
self.assertEqual(token_stream.peek(), Token('num', 123.0))
self.assertEqual(token_stream.next(), Token('num', 123.0))
token_stream = TokenStream(
InputStream(' # comment\n123 abc let a=2 js >=;'))
self.assertEqual(token_stream.next(), Token('num', 123.0))
self.assertEqual(token_stream.next(), Token('var', 'abc'))
self.assertEqual(token_stream.next(), Token('kw', 'let'))
self.assertEqual(token_stream.next(), Token('var', 'a=2'))
self.assertEqual(token_stream.next(), Token('kw', 'js'))
token_stream = TokenStream(InputStream('λ (n) 1'))
self.assertEqual(token_stream.next(), Token("kw", "λ"))
self.assertEqual(token_stream.next(), Token("punc", "("))
self.assertEqual(token_stream.next(), Token("var", "n"))
self.assertEqual(token_stream.next(), Token("punc", ")"))
self.assertEqual(token_stream.next(), Token("num", 1.0))