def test_scanner_with_number(self, mocker):
source = "123 12.23 3+5 13."
on_error_mock = mocker.MagicMock()
scanner = Scanner(source, on_error=on_error_mock)
tokens = scanner.scan_tokens()
assert tokens[0].token_type == TokenType.NUMBER
assert tokens[0].literal == 123.0
assert tokens[1].token_type == TokenType.NUMBER
assert tokens[1].literal == 12.23
assert tokens[2].token_type == TokenType.NUMBER
assert tokens[2].literal == 3.0
assert tokens[3].token_type == TokenType.PLUS
assert tokens[4].token_type == TokenType.NUMBER
assert tokens[4].literal == 5.0
assert tokens[5].token_type == TokenType.NUMBER
assert tokens[5].literal == 13.0
assert not on_error_mock.called
def test_scanner_invalid_identifier(self, mocker):
# The bit of source code below is completely wrong, and identifies and
# numbers in here will not result in valid tokens, but not the tokens you
# would expect. This is not a problem of the scanner, it just does as it's
# told.
source = "123foo_bar bar-stool spam_egg_1.3_chickens"
on_error_mock = mocker.MagicMock()
scanner = Scanner(source, on_error=on_error_mock)
tokens = scanner.scan_tokens()
assert tokens[0].literal == 123.0
assert tokens[1].lexeme == "foo_bar"
assert tokens[1].token_type == TokenType.IDENTIFIER
assert tokens[2].lexeme == "bar"
assert tokens[2].token_type == TokenType.IDENTIFIER
assert tokens[3].token_type == TokenType.MINUS
assert tokens[4].lexeme == "stool"
assert tokens[5].lexeme == "spam_egg_1"
assert tokens[6].token_type == TokenType.DOT
# This token did not consume the 1 before, since that was still part of the
# valid identifier. The dot broke the identifier, and then a number started
assert tokens[7].token_type == TokenType.NUMBER
assert tokens[7].literal == 3.0
assert tokens[8].lexeme == "_chickens"
assert not on_error_mock.called
def test_assignment(self, mocker):
on_scanner_error_mock = mocker.MagicMock()
on_parser_error_mock = mocker.MagicMock()
on_interpret_error_mock = mocker.MagicMock()
lines = [
"var a = 0;",
"var c = a;",
"var b;",
"a = 3 + 6;",
"b = 3 / 6;",
"a = a + b;",
"print(a);",
"a;",
]
expression = "\n".join(lines)
scanner = Scanner(expression, on_error=on_scanner_error_mock)
tokens = scanner.scan_tokens()
parser = Parser(tokens, on_token_error=on_parser_error_mock)
statements = parser.parse()
result = Interpreter().interpret(statements,
on_error=on_interpret_error_mock)
assert result == 9.5
def test_nested_binary_expr(self, create_token_factory, mocker):
""" Test nested binary expressions, 4 * 6 / 2 """
on_scanner_error_mock = mocker.MagicMock()
on_parser_error_mock = mocker.MagicMock()
test_string = "4 * 6 / 2;"
scanner = Scanner(test_string, on_error=on_scanner_error_mock)
tokens = scanner.scan_tokens()
parser = Parser(tokens, on_token_error=on_parser_error_mock)
statements = parser.parse()
expr: Expression = statements[0].expression
assert isinstance(expr, Binary)
assert isinstance(expr.left, Binary)
assert isinstance(expr.right, Literal)
assert expr.operator.token_type == TokenType.SLASH
assert expr.right.value == 2.0
# Left will be 4 * 6
assert expr.left.operator.token_type == TokenType.STAR
assert expr.left.left.value == 4
assert expr.left.right.value == 6
result = Interpreter().visit_binary_expr(expr)
assert result == 12
def run(self, source: str):
logger.debug("Running line", source=source)
scanner = Scanner(source, on_error=self.error)
tokens = scanner.scan_tokens()
for token in tokens:
logger.debug("Running token", token=token)
parser = Parser(tokens, on_token_error=self.token_error)
statements = parser.parse()
if self.had_error:
logger.debug("Error after parsing")
return
resolver = Resolver(interpreter=self.interpreter,
on_error=self.token_error)
resolver.resolve(statements)
# Stop if there was a resolution error.
if self.had_error:
logger.debug("Error after resolving")
return
self.interpreter.interpret(statements, on_error=self.runtime_error)
def test_parser_synchronize(self, mocker):
on_scanner_error_mock = mocker.MagicMock()
on_parser_error_mock = mocker.MagicMock()
lines = [
"var a = 4;",
"print (4+3;",
"a = 5;",
"",
"var a = (3/4",
"print (a);",
]
source = "\n".join(lines)
scanner = Scanner(source, on_error=on_scanner_error_mock)
tokens = scanner.scan_tokens()
parser = Parser(tokens, on_token_error=on_parser_error_mock)
statements = parser.parse()
# We validate that the error is called, the real values are tested in
# test_missing_closing_bracket
assert not on_scanner_error_mock.called
assert on_parser_error_mock.called
# Assert that the third statement is sane and that the parser continues after
# errors
assert isinstance(statements[1], Expression)
assert statements[1].expression.name.line == 3
assert statements[1].expression.name.lexeme == "a"
def test_scanner_operator(self, source, expected_output_list, mocker):
on_error_mock = mocker.MagicMock()
scanner = Scanner(source, on_error=on_error_mock)
tokens = scanner.scan_tokens()
# Retrieve the token_types from the created tokens
token_types = [token.token_type for token in tokens]
assert token_types == expected_output_list
assert not on_error_mock.called
def test_scanner_with_string(self, mocker):
source = '+"This is a String"'
on_error_mock = mocker.MagicMock()
scanner = Scanner(source, on_error=on_error_mock)
tokens = scanner.scan_tokens()
assert tokens[0].token_type == TokenType.PLUS
assert tokens[1].token_type == TokenType.STRING
assert tokens[1].literal == "This is a String"
assert tokens[2].token_type == TokenType.EOF
assert not on_error_mock.called
def test_interpret(self, mocker, expression, result):
on_scanner_error_mock = mocker.MagicMock()
on_parser_error_mock = mocker.MagicMock()
scanner = Scanner(expression, on_error=on_scanner_error_mock)
tokens = scanner.scan_tokens()
parser = Parser(tokens, on_token_error=on_parser_error_mock)
statements = parser.parse()
result = Interpreter().interpret(statements)
assert result == result
def test_scanner_bad_char(self, mocker):
source = "@"
on_error_mock = mocker.MagicMock()
scanner = Scanner(source, on_error=on_error_mock)
tokens = scanner.scan_tokens()
# Invalid character is not added, but the EOF marker is added
assert tokens[0].token_type == TokenType.EOF
assert tokens[0].line == 1
# Assert that our error code has been called
assert on_error_mock.called
on_error_mock.assert_called_once_with(1, "Unexpected character: @")
def test_scanner_with_multiline_string(self, mocker):
source = "This is an \nMulti-\nline-string"
source_input = f'"{source}"'
on_error_mock = mocker.MagicMock()
scanner = Scanner(source_input, on_error=on_error_mock)
tokens = scanner.scan_tokens()
assert tokens[0].token_type == TokenType.STRING
assert tokens[0].literal == source
assert not on_error_mock.called
# We have traveled three lines
assert scanner.line == 3
def test_scanner_with_unterminated_string(self, mocker):
source = '+"This is a Unterminated String'
on_error_mock = mocker.MagicMock()
scanner = Scanner(source, on_error=on_error_mock)
tokens = scanner.scan_tokens()
# Validate that all the default tokens have been added
assert tokens[0].token_type == TokenType.PLUS
assert tokens[1].token_type == TokenType.EOF
# Assert that our error code has been called
assert on_error_mock.called
on_error_mock.assert_called_once_with(1, "Unterminated string.")
def test_scanner(self, mocker):
on_error_mock = mocker.MagicMock()
source = "+-\n*"
scanner = Scanner(source, on_error=on_error_mock)
tokens = scanner.scan_tokens()
assert tokens[0].token_type == TokenType.PLUS
assert tokens[1].token_type == TokenType.MINUS
assert tokens[2].token_type == TokenType.STAR
# the EOF is automatically added
assert tokens[3].token_type == TokenType.EOF
# The newline char (\n) doesn't add a token, but increments the line counter
assert scanner.line == 2
# There mustn't be an error
assert not on_error_mock.called
def test_interpret_error(self, mocker):
on_scanner_error_mock = mocker.MagicMock()
on_parser_error_mock = mocker.MagicMock()
on_interpret_error_mock = mocker.MagicMock()
expression = '0 + "Foo";'
scanner = Scanner(expression, on_error=on_scanner_error_mock)
tokens = scanner.scan_tokens()
parser = Parser(tokens, on_token_error=on_parser_error_mock)
statements = parser.parse()
Interpreter().interpret(statements, on_error=on_interpret_error_mock)
# There will be an error
assert on_interpret_error_mock.called
assert "Operands must be two numbers or two strings" in str(
on_interpret_error_mock.call_args)
def test_for_parser(self, mocker):
""" Test that we parse the for loop correctly """
source = "for(var a = 0; ; a = a + 1) {}"
on_scanner_error_mock = mocker.MagicMock()
on_parser_error_mock = mocker.MagicMock()
scanner = Scanner(source, on_error=on_scanner_error_mock)
tokens = scanner.scan_tokens()
parser = Parser(tokens, on_token_error=on_parser_error_mock)
statements = parser.parse()
assert statements
# Because the condition is missing, it must alway be true:
assert statements[0].statements[1].condition.value is True
assert not on_scanner_error_mock.called
assert not on_parser_error_mock.called
def test_missing_closing_bracket(self, mocker):
on_scanner_error_mock = mocker.MagicMock()
on_parser_error_mock = mocker.MagicMock()
source = "(23+34"
scanner = Scanner(source, on_error=on_scanner_error_mock)
tokens = scanner.scan_tokens()
parser = Parser(tokens, on_token_error=on_parser_error_mock)
statements = parser.parse()
# There will be an error. Scanner must be fine, but the error will be in the
# parser. There will be no statements generated
assert statements == []
assert not on_scanner_error_mock.called
assert on_parser_error_mock.called
on_parser_error_mock.assert_called_once_with(
tokens[-1], "Expect ')' after expression."
)
def test_scanner_identifier(self, mocker):
source = "appelflap or nil if while _foo_bar_1_2"
on_error_mock = mocker.MagicMock()
scanner = Scanner(source, on_error=on_error_mock)
tokens = scanner.scan_tokens()
assert tokens[0].token_type == TokenType.IDENTIFIER
assert tokens[0].lexeme == "appelflap"
assert tokens[1].token_type == TokenType.OR
assert tokens[2].token_type == TokenType.NIL
assert tokens[3].token_type == TokenType.IF
assert tokens[4].token_type == TokenType.WHILE
assert tokens[5].token_type == TokenType.IDENTIFIER
assert tokens[5].lexeme == "_foo_bar_1_2"
assert not on_error_mock.called
def test_invalid_assignment(self, mocker):
"""
An identifiers must start with a letter [a-z], numbers are invalid and
must raise an error
"""
on_scanner_error_mock = mocker.MagicMock()
on_parser_error_mock = mocker.MagicMock()
expression = "var 0123foobar = 34;"
scanner = Scanner(expression, on_error=on_scanner_error_mock)
tokens = scanner.scan_tokens()
parser = Parser(tokens, on_token_error=on_parser_error_mock)
statements = parser.parse()
assert statements == []
assert not on_scanner_error_mock.called
assert on_parser_error_mock.called
on_parser_error_mock.assert_called_once_with(tokens[1], "Expect variable name.")
def test_for_invalid_for_loop_parser(self, mocker):
""" Test that we parse the for loop correctly """
source = "for (var a = 0; a <= 5; a = a + 1) {}"
left_missing = source.replace("(", "")
right_missing = source.replace(")", "")
on_scanner_error_mock = mocker.MagicMock()
on_parser_error_mock = mocker.MagicMock()
scanner = Scanner(source, on_error=on_scanner_error_mock)
tokens = scanner.scan_tokens()
parser = Parser(tokens, on_token_error=on_parser_error_mock)
statements = parser.parse()
assert statements
assert not on_scanner_error_mock.called
assert not on_parser_error_mock.called
# Test a invalid statement, missing (
on_scanner_error_mock = mocker.MagicMock()
on_parser_error_mock = mocker.MagicMock()
scanner = Scanner(left_missing, on_error=on_scanner_error_mock)
tokens = scanner.scan_tokens()
parser = Parser(tokens, on_token_error=on_parser_error_mock)
parser.parse()
assert not on_scanner_error_mock.called
assert on_parser_error_mock.called
assert "Expect '(' after 'for'" in str(on_parser_error_mock.call_args_list[0])
# Test a invalid statement, missing )
on_scanner_error_mock = mocker.MagicMock()
on_parser_error_mock = mocker.MagicMock()
scanner = Scanner(right_missing, on_error=on_scanner_error_mock)
tokens = scanner.scan_tokens()
parser = Parser(tokens, on_token_error=on_parser_error_mock)
parser.parse()
assert not on_scanner_error_mock.called
assert on_parser_error_mock.called
assert "Expect ')' after for clauses" in str(
on_parser_error_mock.call_args_list[0]
)
def test_scanner_bad_char_without_callback(self):
source = "@"
scanner = Scanner(source)
with pytest.raises(KeyError):
scanner.scan_tokens()