def testNewParser(self):
program = lex(
"print 'Hello World!'\n"
"print 42"
)
ast = parse(program)
def testBadParse(self):
program = 'if 1<2\n pass\n'
tokens = lex(program)
def ifParse():
IfStatement.parse(tokens)
self.failUnlessRaises(error.SyntaxError, ifParse)
def testIndentation(self):
result = lex(util.source('''
0
4
4
8
0
2
6'''))
EOS = END_OF_STATEMENT
BB = BEGIN_BLOCK
EB = END_BLOCK
EOF = END_OF_FILE
expected = [
'0', EOS, BB, '4', EOS, '4', EOS,
BB, '8', EOS, EB, EB, EOS, '0', EOS,
BB, '2', EOS, BB, '6', EOS, EB, EB, EOS, EOF
]
self.failUnlessEqual(len(result), len(expected))
self.failUnlessEqual(result, expected)
def testSemantic(self):
'''Also tests variable declaration/expression things.
The basic jist is that, once semantic testing has been done,
an Identifier should no longer be an Identifier; it should be
a VariableExpression.
'''
from ast.vardecl import VarDecl
from ast.variableexpression import VariableExpression
decl = VarDecl('testvar', (0, '<test>'), vartypes.IntType)
scope = Scope(parent=None)
scope['testvar'] = decl
tokens = lex('testvar nonexist')
expr1 = Identifier.parse(tokens)
expr2 = Identifier.parse(tokens)
result = expr1.semantic(scope)
self.failUnlessEqual(type(result), VariableExpression)
self.failUnlessEqual(result.variable, decl)
self.failUnlessRaises(
error.NameError,
lambda: expr2.semantic(scope)
)
def testBadParse(self):
tokens = lex('()')
self.failUnlessRaises(
error.SyntaxError,
lambda: ParenthExpression.parse(tokens)
)
def testBadSemantic(self):
program = "var x = 5 * 'bork'\n"
self.failUnlessRaises(
error.TypeError,
lambda: semantic(parse(lex(program)))
)
def testGoodStringParse(self):
tokens = lex("'print' \"print\"")
result = LiteralExpression.parse(tokens)
self.failUnless(isinstance(result, StringLiteral))
result2 = LiteralExpression.parse(tokens)
self.failUnless(isinstance(result, StringLiteral))
def testGoodParse(self):
program = util.source('''
def foo():
pass
''')
result = FunctionDecl.parse(lex(program))
assert isinstance(result, FunctionDecl), result
def testComments(self):
result = lex(util.source('''
this is tokens!
# this is a comment
this is not
'''))
assert '#' not in result
def testGoodProgram(self):
source = lex(
"print 'Hello World!'"
)
ast = parse(source)
result = semantic.semantic(ast)
def testTypeMismatch2(self):
program = util.source('''
def foo(i as int) as void:
return i
''')
self.failUnlessRaises(
error.TypeError,
lambda: semantic(parse(lex(program)))
)
def testBadParse(self):
program = util.source('''
def foo():
''')
self.failUnlessRaises(
error.SyntaxError,
lambda: FunctionDecl.parse(lex(program))
)
def testTypeMismatch(self):
ast = parse(lex('var x as string = 42'))
assert ast is not None
self.failUnlessRaises(
error.TypeError,
lambda: semantic(ast)
)
def doIt():
result = lex(util.source('''
0
4
4
2
''')
)
def testLexKeyword(self):
result = lex('int print abc string char if 0')
self.failUnlessEqual(result[0].type, 'keyword')
self.failUnlessEqual(result[1].type, 'keyword')
self.failUnlessEqual(result[2].type, 'identifier')
self.failUnlessEqual(result[3].type, 'keyword')
self.failUnlessEqual(result[4].type, 'keyword')
self.failUnlessEqual(result[5].type, 'keyword')
self.failUnlessEqual(result[6].type, 'literal')
def testABugAndyFound(self):
from nine.lexer import lex
from nine.parser import parse
from nine.semantic import semantic
result = semantic(parse(lex(util.source('''
if true:
print 'True!'
print 'This caused a spurious syntax error because there is no END_OF_STATEMENT after the dedent!'
'''))))
def testParseAttributeNoParameters(self):
source = '[Thingie]'
tok = lexer.lex(source)
result = Attribute.parse(tok)
self.assertTrue(isinstance(result, Attribute))
self.assertTrue(isinstance(result.className, Identifier))
self.assertEqual(result.className.name, 'Thingie')
self.assertEqual(result.params, [])
def testParseArgs(self):
tokens = lex(util.source('''
def foo(x as string, y as int, z as boolean):
print x
print y
print z
'''))
result = FunctionDecl.parse(tokens)
assert isinstance(result, FunctionDecl), tokens.peek()
def testMissingCloseBrace(self):
source = util.source('''
hey look we forgot to close this (!
oops!
''')
self.assertRaises(
error.LexError,
lambda: lex(source)
)
def testRenameIdentifier(self):
from nine.lexer import lex
from ast.qualifiedname import QualifiedName
name = QualifiedName.parse(lex('Foo.Bar.Baz'))
newName = Attribute.renameIdentifier(name)
self.assertEqual(newName.lhs, name.lhs)
self.assertEqual(newName.lhs.rhs.name, name.lhs.rhs.name)
self.assertEqual(newName.rhs.name, 'BazAttribute')
def testSemantic(self):
tokens = lex('"abcd"+3')
expr = AddExpression.parse(tokens)
from nine.scope import Scope
scope = Scope(parent=None)
from nine import error
self.failUnlessRaises(error.TypeError,
lambda: expr.semantic(scope)
)
def testTypeMismatch1(self):
program = util.source('''
def foo(i as int) as int:
return "This isn't an int!!"
''')
ast = parse(lex(program))
self.failUnlessRaises(
error.TypeError,
lambda: semantic(ast)
)
def testSemantic(self):
# Also tests Identifier and name resolution.
decl = VarDecl('testvar', (0, '<test>'), vartypes.IntType)
scope = Scope(parent=None)
scope['testvar'] = decl
tokens = lex('testvar')
expr = Identifier.parse(tokens)
result = expr.semantic(scope)
self.failUnlessEqual(type(result), VariableExpression)
self.failUnlessEqual(result.variable, decl)
def testSemantic2(self):
from nine.parser import parse
from nine.semantic import semantic
program = util.source('''
var x = 42
var y = 'Hello!'
print x
print y
''')
result = semantic(parse(lex(program)))
assert result is not None
def testTypeInfer(self):
from ast import vartypes
program = util.source('''
var x = 19
var z = 'Hello!'
''')
ast = parse(lex(program))
st = semantic(ast)
assert st[0].type is vartypes.IntType, (st[0].type, vartypes.IntType)
assert st[1].type is vartypes.StringType, (st[1].type, vartypes.StringType)
def testSemantic(self):
from nine.parser import parse
from nine.semantic import semantic
tokens = lex(util.source('''
def foo():
print 'foo!'
foo()
'''))
ast = parse(tokens)
st = semantic(ast)
def testSemantic(self):
tokens = lex('var x as int')
scope = Scope(parent=None)
decl = VarDecl.parse(tokens)
result = decl.semantic(scope)
self.failUnless(isinstance(result, VarDecl), result)
self.failUnless('x' in scope)
# x is already defined in the scope, this should fail, as it is a duplicate
self.failUnlessRaises(Exception,
lambda: tokens.semantic(scope)
)
def testNameResolution(self):
program = util.source('''
class A(B):
pass
class B(A):
pass
''')
ast = parse(lex(program))
globalScope = Scope()
semantic.collectNames(ast, globalScope)
semantic.resolveNames(ast, globalScope)
def testLineTracking(self):
tokens = lex(util.source('''
Line1
Line3
Line4
Line6
'''))
# get the "Line6" token
line6 = [tok for tok in tokens if tok == "Line6"][0]
self.failUnlessEqual(line6.line, 6)
def semanticProgram(program, assemblies=[]):
from ast.namespace import Namespace
from ast.vartypes import Type
driver = Driver()
globalNs = Namespace('')
for asm in assemblies:
driver._scanAssembly(globalNs, asm)
driver.fixPrimitives()
globalScope = semantic.makeGlobalScope(globalNs.symbols)
return semantic.semantic(parse(lex(program)), globalScope)
