def test_parsing_hash_literals_string_keys(self):
input = '{"one": 1, "two": 2, "three": 3}'
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
stmt = program.Statements[0]
hash = stmt.ExpressionValue
if not hash:
self.fail('exp is not ast.HashLiteral. got=%s' % stmt.Expression)
if len(hash.Pairs) != 3:
self.fail('hash.Pairs has wrong length. got=%s' % len(hash.Pairs))
expected = {
'one': 1,
'two': 2,
'three': 3,
}
for key, value in hash.Pairs:
expectedValue = expected[key.Value]
testIntegerLiteral(self, value, expectedValue)
def test_parsing_prefix_expressions(self):
@dataclass
class Prefix():
input: str
operator: str
value: Any
prefixTests = [
Prefix('!5;', '!', 5),
Prefix('-15', '-', 15),
Prefix('!true;', '!', True),
Prefix('!false;', '!', False),
]
for tt in prefixTests:
lex = lexer.New(tt.input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
if len(program.Statements) != 1:
self.fail('program.Statements does not contain %s statements. got=%s' %
(lex, len(program.Statements)))
stmt = program.Statements[0]
if stmt is None:
self.fail('program.Statements[0] is not ast.ExpressionStatement. got=%s' % stmt)
exp = stmt.ExpressionValue
if exp is None:
self.fail('exp not *ast.PrefixExpression. got=%s' % stmt.ExpressionValue)
if exp.Operator != tt.operator:
self.fail('exp.Operator is not \'%s\'. got=%s' % (tt.operator, exp.Operator))
if not testLiteralExpression(self, exp.Right, tt.value):
continue
def test_call_expression_parsing(self):
input = 'add(1, 2 * 3, 4 + 5);'
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
if len(program.Statements) != 1:
self.fail('program.Statements does not contain %s statements. got=%s\n' %
(1, len(program.Statements)))
stmt = program.Statements[0]
if not stmt:
self.fail('stmt is not ast.ExpressionStatement. got=%s' % program.Statements[0])
exp = stmt.ExpressionValue
if not exp:
self.fail('stmt.Expression is not ast.CallExpression. got=%s' % exp)
if not testIdentifier(self, exp.Function, 'add'):
return
if len(exp.Arguments) != 3:
self.fail('wrong length of arguments. got=%s' % len(exp.Arguments))
testLiteralExpression(self, exp.Arguments[0], 1)
testInfixExpression(self, exp.Arguments[1], 2, '*', 3)
testInfixExpression(self, exp.Arguments[2], 4, '+', 5)
def test_function_parameter_parsing(self):
@dataclass
class Test():
input: str
expectedParams: List[str]
tests: List[Test] = [
Test(input='fn() {};', expectedParams=[]),
Test(input='fn(x) {};', expectedParams=['x']),
Test(input='fn(x, y, z) {};', expectedParams=['x', 'y', 'z']),
]
for tt in tests:
lex = lexer.New(tt.input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
stmt = program.Statements[0]
function = stmt.ExpressionValue
if len(function.Parameters) != len(tt.expectedParams):
self.fail('length parameters wrong. want %d, got=%s\n' % (len(tt.expectedParams),
len(function.Parameters)))
for i, ident in enumerate(tt.expectedParams):
testLiteralExpression(self, function.Parameters[i], ident)
def Start() -> None:
env = object.NewEnvironment()
macroEnv = object.NewEnvironment()
while True:
try:
line = input(PROMPT)
except KeyboardInterrupt:
print('\nKeyboardInterrupt')
continue
if line == 'exit()':
break
lex = lexer.New(line)
p = parser.New(lex)
program = p.ParseProgram()
if len(p.Errors()) is not 0:
printParserErrors(p.Errors())
continue
evaluator.DefineMacros(program, macroEnv)
expanded = evaluator.ExpandMacros(program, macroEnv)
evaluator.Eval(expanded, env)
def test_parsing_hash_literals_with_expressions(self):
input = '{"one": 0 + 1, "two": 10 - 8, "three": 15 / 5}'
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
stmt = program.Statements[0]
hash = stmt.ExpressionValue
if not hash:
self.fail('exp is not ast.HashLiteral. got=%s' % stmt.ExpressionValue)
if len(hash.Pairs) != 3:
self.fail('hash.Pairs has wrong length. got=%s' % len(hash.Pairs))
tests = {
'one': lambda e: testInfixExpression(self, e, 0, '+', 1),
'two': lambda e: testInfixExpression(self, e, 10, '-', 8),
'three': lambda e: testInfixExpression(self, e, 15, '/', 5)
}
for key, value in hash.Pairs:
literal = key
if not literal:
print('key is not ast.StringLiteral. got=%s' % key)
continue
testFunc = tests[literal.Value]
if not testFunc:
print('No test function for key %s found' % literal)
continue
testFunc(value)
def test_parsing_infix_expressions(self):
@dataclass
class Infix():
input: str
leftValue: Any
operator: str
rightValue: Any
infixTests: List[Infix] = [
Infix('5 + 5;', 5, '+', 5),
Infix('5 - 5;', 5, '-', 5),
Infix('5 * 5;', 5, '*', 5),
Infix('5 / 5;', 5, '/', 5),
Infix('5 > 5;', 5, '>', 5),
Infix('5 < 5;', 5, '<', 5),
Infix('5 == 5;', 5, '==', 5),
Infix('5 != 5;', 5, '!=', 5),
Infix('true == true', True, '==', True),
Infix('true != false', True, '!=', False),
Infix('false == false', False, '==', False),
]
for tt in infixTests:
lex = lexer.New(tt.input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
if len(program.Statements) != 1:
self.fail('program.Statements does not contain %s statements. got=%s\n' %
(1, len(program.Statements)))
stmt = program.Statements[0]
if not stmt:
self.fail('program.Statements[0] is not ast.ExpressionStatement. got=%s' %
program.Statements[0])
exp = stmt.ExpressionValue
if not exp:
self.fail('exp is not ast.InfixExpression. got=%s' % stmt.ExpressionValue)
if not testInfixExpression(self, stmt.ExpressionValue, tt.leftValue, tt.operator,
tt.rightValue):
continue
if not testLiteralExpression(self, exp.Left, tt.leftValue):
continue
if exp.Operator != tt.operator:
self.fail('exp.Operator is not \'%s\'. got=%s' % (tt.operator, exp.Operator))
if not testLiteralExpression(self, exp.Right, tt.rightValue):
continue
def test_string_literal_expression(self):
input = '"hello world";'
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
stmt = program.Statements[0]
literal = stmt.ExpressionValue
if not literal:
self.fail('exp not *ast.StringLiteral. got=%s' % stmt.Expression)
if literal.Value != 'hello world':
self.fail('literal.Value not %s. got=%s' % ('hello world', literal.Value))
def test_operator_precedence_parsing(self):
@dataclass
class Test():
input: str
expected: str
tests: List[Test] = [
Test('-a * b', '((-a) * b)'),
Test('!-a', '(!(-a))'),
Test('a + b + c', '((a + b) + c)'),
Test('a + b - c', '((a + b) - c)'),
Test('a * b * c', '((a * b) * c)'),
Test('a * b / c', '((a * b) / c)'),
Test('a + b / c', '(a + (b / c))'),
Test('a + b * c + d / e - f', '(((a + (b * c)) + (d / e)) - f)'),
Test('3 + 4; -5 * 5', '(3 + 4)((-5) * 5)'),
Test('5 > 4 == 3 < 4', '((5 > 4) == (3 < 4))'),
Test('5 < 4 != 3 > 4', '((5 < 4) != (3 > 4))'),
Test('3 + 4 * 5 == 3 * 1 + 4 * 5', '((3 + (4 * 5)) == ((3 * 1) + (4 * 5)))'),
Test('true', 'true'),
Test('false', 'false'),
Test('3 > 5 == false', '((3 > 5) == false)'),
Test('3 < 5 == true', '((3 < 5) == true)'),
Test('1 + (2 + 3) + 4', '((1 + (2 + 3)) + 4)'),
Test('(5 + 5) * 2', '((5 + 5) * 2)'),
Test('2 / (5 + 5)', '(2 / (5 + 5))'),
Test('-(5 + 5)', '(-(5 + 5))'),
Test('!(true == true)', '(!(true == true))'),
Test('a + add(b * c) + d', '((a + add((b * c))) + d)'),
Test('add(a, b, 1, 2 * 3, 4 + 5, add(6, 7 * 8))',
'add(a, b, 1, (2 * 3), (4 + 5), add(6, (7 * 8)))'),
Test('add(a + b + c * d / f + g)', 'add((((a + b) + ((c * d) / f)) + g))'),
Test('a * [1, 2, 3, 4][b * c] * d', '((a * ([1, 2, 3, 4][(b * c)])) * d)'),
Test('add(a * b[2], b[1], 2 * [1, 2][1])',
'add((a * (b[2])), (b[1]), (2 * ([1, 2][1])))'),
]
for tt in tests:
lex = lexer.New(tt.input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
actual = program.String()
if actual != tt.expected:
self.fail('expected=%s, got=%s' % (tt.expected, actual))
def test_parsing_index_expressions(self):
input = 'myArray[1 + 1]'
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
stmt = program.Statements[0]
indexExp = stmt.ExpressionValue
if not indexExp:
self.fail('exp not *ast.IndexExpression. got=%s' % stmt.Expression)
if not testIdentifier(self, indexExp.Left, 'myArray'):
return
if not testInfixExpression(self, indexExp.Index, 1, '+', 1):
return
def test_identifier_expression(self):
input = 'foobar;'
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
if len(program.Statements) != 1:
self.fail('program has not enough statements. got=%s' % len(program.Statements))
stmt = program.Statements[0]
ident = stmt.ExpressionValue
if ident is None:
self.fail('exp not *ast.Identifier. got=%s' % stmt.ExpressionValue)
if ident.Value != 'foobar':
self.fail('exp not *ast.Identifier. got=%s' % stmt.Expression)
if ident.TokenLiteral() != 'foobar':
self.fail('ident.TokenLiteral not %s. got=%s' % ('foobar', ident.TokenLiteral()))
def test_parsing_array_literals(self):
input = '[1, 2 * 2, 3 + 3]'
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
stmt = program.Statements[0]
array = stmt.ExpressionValue
if not array:
self.fail('exp not ast.ArrayLiteral. got=%s' % stmt.ExpressionValue)
if len(array.Elements) != 3:
self.fail('len(array.Elements) not 3. got=%s' % len(array.Elements))
testIntegerLiteral(self, array.Elements[0], 1)
testInfixExpression(self, array.Elements[1], 2, '*', 2)
testInfixExpression(self, array.Elements[2], 3, '+', 3)
def test_integer_literal_expression(self):
input = '5;'
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
if len(program.Statements) != 1:
self.fail('program has not enough statements. got=%s' % len(program.Statements))
stmt = program.Statements[0]
if stmt is None:
self.fail('program.Statements[0] is not ast.ExpressionStatement. got=%s' % stmt)
literal = stmt.ExpressionValue
if literal is None:
self.fail('exp not *ast.IntegerLiteral. got=%s' % stmt.ExpressionValue)
if literal.Value != 5:
self.fail('literal.Value not %s. got=%s' % (5, stmt.Expression))
if literal.TokenLiteral() != '5':
self.fail('literal.TokenLiteral not %s. got=%s' % ('5', literal.TokenLiteral()))
def test_function_literal_parsing(self):
input = 'fn(x, y) { x + y; }'
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
if len(program.Statements) != 1:
self.fail('program.Statements does not contain %s statements. got=%s\n' %
(1, len(program.Statements)))
stmt = program.Statements[0]
if not stmt:
self.fail('program.Statements[0] is not ast.ExpressionStatement. got=%s' %
program.Statements[0])
function = stmt.ExpressionValue
if not function:
self.fail('stmt.Expression is not ast.FunctionLiteral. got=%s' % stmt.ExpressionValue)
if len(function.Parameters) != 2:
self.fail(
'function literal parameters wrong. want 2, got=%s\n' % len(function.Parameters))
testLiteralExpression(self, function.Parameters[0], 'x')
testLiteralExpression(self, function.Parameters[1], 'y')
if len(function.Body.Statements) != 1:
self.fail('function.Body.Statements has not 1 statements. got=%s\n' % len(
function.Body.Statements))
bodyStmt = function.Body.Statements[0]
if not bodyStmt:
self.fail('function body stmt is not ast.ExpressionStatement. got=%s' %
function.Body.Statements[0])
testInfixExpression(self, bodyStmt.ExpressionValue, 'x', '+', 'y')
def test_return_statetments(self):
input = '''
return 5;
return 10;
return 993322;
'''
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
if len(program.Statements) != 3:
self.fail('program.Statements does not contain 3 statements. got=%s' % len(
program.Statements))
for stmt in program.Statements:
if type(stmt) != ast.ReturnStatement:
self.fail('stmt not *ast.returnStatement. got=%s' % stmt)
continue
returnStmt = stmt
if returnStmt.TokenLiteral() != 'return':
self.fail(
'returnStmt.TokenLiteral not \'return\', got %s' % returnStmt.TokenLiteral())
def test_let_statements(self):
@dataclass
class Test():
input: str
expectedIdentifier: str
expectedValue: Any
tests: List[Test] = [
Test('let x = 5;', 'x', 5),
Test('let y = true;', 'y', True),
Test('let foobar = y;', 'foobar', 'y'),
]
# let x = 5;
# let y = 10;
# let foobar = 838383;
for tt in tests:
lex = lexer.New(tt.input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
if program is None:
self.fail('ParseProgram() returned None')
if len(program.Statements) != 1:
self.fail('program.Statements does not contain 1 statements. got=%s' % len(
program.Statements))
stmt = program.Statements[0]
if not testLetStatement(self, stmt, tt.expectedIdentifier):
continue
val = stmt.Value
if not testLiteralExpression(self, val, tt.expectedValue):
continue
def main() -> None:
argparser = argparse.ArgumentParser(description='')
argparser.add_argument('infile', nargs='?', type=argparse.FileType('r'))
args = argparser.parse_args()
if args.infile:
body = args.infile.read()
if body:
env = object.NewEnvironment()
lex = lexer.New(body)
p = parser.New(lex)
program = p.ParseProgram()
if len(p.Errors()) is not 0:
for msg in p.Errors():
print('\t' + msg)
return
evaluator.Eval(program, env)
else:
user = getpass.getuser()
print('Hello {}! This is the Monkey programming language!\n'.format(
user),
end='')
print('Feel free to type in commands\n', end='')
repl.Start()
def test_macro_literal_parsing(self):
input = 'macro(x, y) { x + y; }'
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
if len(program.Statements) != 1:
self.fail('program.Statements does not contain %s statements. got=%s\n' %
(1, len(program.Statements)))
stmt = program.Statements[0]
if not stmt:
self.fail('statement is not ast.ExpressionStatement. got=%s' % program.Statements[0])
macro = stmt.ExpressionValue
if not macro:
self.fail('stmt.Expression is not ast.MacroLiteral. got=%s' % stmt.ExpressionValue)
if len(macro.Parameters) != 2:
self.fail('macro literal parameters wrong. want 2, got=%s\n' % len(macro.Parameters))
testLiteralExpression(self, macro.Parameters[0], 'x')
testLiteralExpression(self, macro.Parameters[1], 'y')
if len(macro.Body.Statements) != 1:
self.fail(
'macro.Body.Statements has not 1 statements. got=%s\n' % len(macro.Body.Statements))
bodyStmt = macro.Body.Statements[0]
if not bodyStmt:
self.fail(
'macro body stmt is not ast.ExpressionStatement. got=%s' % macro.Body.Statements[0])
testInfixExpression(self, bodyStmt.ExpressionValue, 'x', '+', 'y')
def test_if_else_expression(self):
input = 'if (x < y) { x } else { y }'
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
checkParserErrors(self, p)
if len(program.Statements) != 1:
self.fail('program.Statements does not contain %s statements. got=%s\n' %
(lex, len(program.Statements)))
stmt = program.Statements[0]
if not stmt:
self.fail('program.Statements[0] is not ast.ExpressionStatement. got=%s' % stmt)
exp = stmt.ExpressionValue
if not exp:
self.fail('stmt.Expression is not ast.IfExpression. got=%s' % stmt.Expression)
if not testInfixExpression(self, exp.Condition, 'x', '<', 'y'):
return
if len(exp.Consequence.Statements) != 1:
self.fail('consequence is not 1 statements. got=%s\n' % len(exp.Consequence.Statements))
consequence = exp.Consequence.Statements[0]
if not consequence:
self.fail('Statements[0] is not ast.ExpressionStatement. got=%s\n' %
exp.Consequence.Statements[0])
if not testIdentifier(self, consequence.ExpressionValue, 'x'):
return
if not exp.Alternative:
self.fail('exp.Alternative.Statements was not nil. got=%s' % exp.Alternative)
def test_next_token(self):
input = '''let five = 5;
let ten = 10;
let add = fn(x, y) {
x + y;
};
let result = add(five, ten);
!-/*5;
5 < 10 > 5;
if (5 < 10) {
return true;
} else {
return false;
}
10 == 10;
10 != 9;
"foobar"
"foo bar"
[1, 2];
{"foo": "bar"}
macro(x, y) { x + y; };
'''
tests = [
[token.LET, 'let'],
[token.IDENT, 'five'],
[token.ASSIGN, '='],
[token.INT, '5'],
[token.SEMICOLON, ';'],
[token.LET, 'let'],
[token.IDENT, 'ten'],
[token.ASSIGN, '='],
[token.INT, '10'],
[token.SEMICOLON, ';'],
[token.LET, 'let'],
[token.IDENT, 'add'],
[token.ASSIGN, '='],
[token.FUNCTION, 'fn'],
[token.LPAREN, '('],
[token.IDENT, 'x'],
[token.COMMA, ','],
[token.IDENT, 'y'],
[token.RPAREN, ')'],
[token.LBRACE, '{'],
[token.IDENT, 'x'],
[token.PLUS, '+'],
[token.IDENT, 'y'],
[token.SEMICOLON, ';'],
[token.RBRACE, '}'],
[token.SEMICOLON, ';'],
[token.LET, 'let'],
[token.IDENT, 'result'],
[token.ASSIGN, '='],
[token.IDENT, 'add'],
[token.LPAREN, '('],
[token.IDENT, 'five'],
[token.COMMA, ','],
[token.IDENT, 'ten'],
[token.RPAREN, ')'],
[token.SEMICOLON, ';'],
[token.BANG, '!'],
[token.MINUS, '-'],
[token.SLASH, '/'],
[token.ASTERISK, '*'],
[token.INT, '5'],
[token.SEMICOLON, ';'],
[token.INT, '5'],
[token.LT, '<'],
[token.INT, '10'],
[token.GT, '>'],
[token.INT, '5'],
[token.SEMICOLON, ';'],
[token.IF, 'if'],
[token.LPAREN, '('],
[token.INT, '5'],
[token.LT, '<'],
[token.INT, '10'],
[token.RPAREN, ')'],
[token.LBRACE, '{'],
[token.RETURN, 'return'],
[token.TRUE, 'true'],
[token.SEMICOLON, ';'],
[token.RBRACE, '}'],
[token.ELSE, 'else'],
[token.LBRACE, '{'],
[token.RETURN, 'return'],
[token.FALSE, 'false'],
[token.SEMICOLON, ';'],
[token.RBRACE, '}'],
[token.INT, '10'],
[token.EQ, '=='],
[token.INT, '10'],
[token.SEMICOLON, ';'],
[token.INT, '10'],
[token.NOT_EQ, '!='],
[token.INT, '9'],
[token.SEMICOLON, ';'],
[token.STRING, 'foobar'],
[token.STRING, 'foo bar'],
[token.LBRACKET, '['],
[token.INT, '1'],
[token.COMMA, ','],
[token.INT, '2'],
[token.RBRACKET, ']'],
[token.SEMICOLON, ';'],
[token.LBRACE, '{'],
[token.STRING, 'foo'],
[token.COLON, ':'],
[token.STRING, 'bar'],
[token.RBRACE, '}'],
[token.MACRO, 'macro'],
[token.LPAREN, '('],
[token.IDENT, 'x'],
[token.COMMA, ','],
[token.IDENT, 'y'],
[token.RPAREN, ')'],
[token.LBRACE, '{'],
[token.IDENT, 'x'],
[token.PLUS, '+'],
[token.IDENT, 'y'],
[token.SEMICOLON, ';'],
[token.RBRACE, '}'],
[token.SEMICOLON, ';'],
[token.EOF, ''],
]
lex = lexer.New(input)
for i, tt in enumerate(tests):
tok = lex.NextToken()
if tok.Type != tt[0]:
self.fail('tests[%d] - tokentype wrong. expected=\'%s\', got=\'%s\'' % (i, tt[0],
tok.Type))
if tok.Literal != tt[1]:
self.fail('tests[%d] - tokenliteral wrong. expected=\'%s\', got=\'%s\'' %
(i, tt[1], tok.Literal))
def testEval(input: str) -> object.Object:
lex = lexer.New(input)
p = parser.New(lex)
program = p.ParseProgram()
env = object.NewEnvironment()
return evaluator.Eval(program, env)
def testParseProgram(input: str) -> ast.Program:
lex = lexer.New(input)
p = parser.New(lex)
return p.ParseProgram()