def testNewParser(self): program = lex( "print 'Hello World!'\n" "print 42" ) ast = parse(program)
def testBadParse(self): program = 'if 1<2\n pass\n' tokens = lex(program) def ifParse(): IfStatement.parse(tokens) self.failUnlessRaises(error.SyntaxError, ifParse)
def testIndentation(self): result = lex(util.source(''' 0 4 4 8 0 2 6''')) EOS = END_OF_STATEMENT BB = BEGIN_BLOCK EB = END_BLOCK EOF = END_OF_FILE expected = [ '0', EOS, BB, '4', EOS, '4', EOS, BB, '8', EOS, EB, EB, EOS, '0', EOS, BB, '2', EOS, BB, '6', EOS, EB, EB, EOS, EOF ] self.failUnlessEqual(len(result), len(expected)) self.failUnlessEqual(result, expected)
def testSemantic(self): '''Also tests variable declaration/expression things. The basic jist is that, once semantic testing has been done, an Identifier should no longer be an Identifier; it should be a VariableExpression. ''' from ast.vardecl import VarDecl from ast.variableexpression import VariableExpression decl = VarDecl('testvar', (0, '<test>'), vartypes.IntType) scope = Scope(parent=None) scope['testvar'] = decl tokens = lex('testvar nonexist') expr1 = Identifier.parse(tokens) expr2 = Identifier.parse(tokens) result = expr1.semantic(scope) self.failUnlessEqual(type(result), VariableExpression) self.failUnlessEqual(result.variable, decl) self.failUnlessRaises( error.NameError, lambda: expr2.semantic(scope) )
def testBadParse(self): tokens = lex('()') self.failUnlessRaises( error.SyntaxError, lambda: ParenthExpression.parse(tokens) )
def testBadSemantic(self): program = "var x = 5 * 'bork'\n" self.failUnlessRaises( error.TypeError, lambda: semantic(parse(lex(program))) )
def testGoodStringParse(self): tokens = lex("'print' \"print\"") result = LiteralExpression.parse(tokens) self.failUnless(isinstance(result, StringLiteral)) result2 = LiteralExpression.parse(tokens) self.failUnless(isinstance(result, StringLiteral))
def testGoodParse(self): program = util.source(''' def foo(): pass ''') result = FunctionDecl.parse(lex(program)) assert isinstance(result, FunctionDecl), result
def testComments(self): result = lex(util.source(''' this is tokens! # this is a comment this is not ''')) assert '#' not in result
def testGoodProgram(self): source = lex( "print 'Hello World!'" ) ast = parse(source) result = semantic.semantic(ast)
def testTypeMismatch2(self): program = util.source(''' def foo(i as int) as void: return i ''') self.failUnlessRaises( error.TypeError, lambda: semantic(parse(lex(program))) )
def testBadParse(self): program = util.source(''' def foo(): ''') self.failUnlessRaises( error.SyntaxError, lambda: FunctionDecl.parse(lex(program)) )
def testTypeMismatch(self): ast = parse(lex('var x as string = 42')) assert ast is not None self.failUnlessRaises( error.TypeError, lambda: semantic(ast) )
def doIt(): result = lex(util.source(''' 0 4 4 2 ''') )
def testLexKeyword(self): result = lex('int print abc string char if 0') self.failUnlessEqual(result[0].type, 'keyword') self.failUnlessEqual(result[1].type, 'keyword') self.failUnlessEqual(result[2].type, 'identifier') self.failUnlessEqual(result[3].type, 'keyword') self.failUnlessEqual(result[4].type, 'keyword') self.failUnlessEqual(result[5].type, 'keyword') self.failUnlessEqual(result[6].type, 'literal')
def testABugAndyFound(self): from nine.lexer import lex from nine.parser import parse from nine.semantic import semantic result = semantic(parse(lex(util.source(''' if true: print 'True!' print 'This caused a spurious syntax error because there is no END_OF_STATEMENT after the dedent!' '''))))
def testParseAttributeNoParameters(self): source = '[Thingie]' tok = lexer.lex(source) result = Attribute.parse(tok) self.assertTrue(isinstance(result, Attribute)) self.assertTrue(isinstance(result.className, Identifier)) self.assertEqual(result.className.name, 'Thingie') self.assertEqual(result.params, [])
def testParseArgs(self): tokens = lex(util.source(''' def foo(x as string, y as int, z as boolean): print x print y print z ''')) result = FunctionDecl.parse(tokens) assert isinstance(result, FunctionDecl), tokens.peek()
def testMissingCloseBrace(self): source = util.source(''' hey look we forgot to close this (! oops! ''') self.assertRaises( error.LexError, lambda: lex(source) )
def testRenameIdentifier(self): from nine.lexer import lex from ast.qualifiedname import QualifiedName name = QualifiedName.parse(lex('Foo.Bar.Baz')) newName = Attribute.renameIdentifier(name) self.assertEqual(newName.lhs, name.lhs) self.assertEqual(newName.lhs.rhs.name, name.lhs.rhs.name) self.assertEqual(newName.rhs.name, 'BazAttribute')
def testSemantic(self): tokens = lex('"abcd"+3') expr = AddExpression.parse(tokens) from nine.scope import Scope scope = Scope(parent=None) from nine import error self.failUnlessRaises(error.TypeError, lambda: expr.semantic(scope) )
def testTypeMismatch1(self): program = util.source(''' def foo(i as int) as int: return "This isn't an int!!" ''') ast = parse(lex(program)) self.failUnlessRaises( error.TypeError, lambda: semantic(ast) )
def testSemantic(self): # Also tests Identifier and name resolution. decl = VarDecl('testvar', (0, '<test>'), vartypes.IntType) scope = Scope(parent=None) scope['testvar'] = decl tokens = lex('testvar') expr = Identifier.parse(tokens) result = expr.semantic(scope) self.failUnlessEqual(type(result), VariableExpression) self.failUnlessEqual(result.variable, decl)
def testSemantic2(self): from nine.parser import parse from nine.semantic import semantic program = util.source(''' var x = 42 var y = 'Hello!' print x print y ''') result = semantic(parse(lex(program))) assert result is not None
def testTypeInfer(self): from ast import vartypes program = util.source(''' var x = 19 var z = 'Hello!' ''') ast = parse(lex(program)) st = semantic(ast) assert st[0].type is vartypes.IntType, (st[0].type, vartypes.IntType) assert st[1].type is vartypes.StringType, (st[1].type, vartypes.StringType)
def testSemantic(self): from nine.parser import parse from nine.semantic import semantic tokens = lex(util.source(''' def foo(): print 'foo!' foo() ''')) ast = parse(tokens) st = semantic(ast)
def testSemantic(self): tokens = lex('var x as int') scope = Scope(parent=None) decl = VarDecl.parse(tokens) result = decl.semantic(scope) self.failUnless(isinstance(result, VarDecl), result) self.failUnless('x' in scope) # x is already defined in the scope, this should fail, as it is a duplicate self.failUnlessRaises(Exception, lambda: tokens.semantic(scope) )
def testNameResolution(self): program = util.source(''' class A(B): pass class B(A): pass ''') ast = parse(lex(program)) globalScope = Scope() semantic.collectNames(ast, globalScope) semantic.resolveNames(ast, globalScope)
def testLineTracking(self): tokens = lex(util.source(''' Line1 Line3 Line4 Line6 ''')) # get the "Line6" token line6 = [tok for tok in tokens if tok == "Line6"][0] self.failUnlessEqual(line6.line, 6)
def semanticProgram(program, assemblies=[]): from ast.namespace import Namespace from ast.vartypes import Type driver = Driver() globalNs = Namespace('') for asm in assemblies: driver._scanAssembly(globalNs, asm) driver.fixPrimitives() globalScope = semantic.makeGlobalScope(globalNs.symbols) return semantic.semantic(parse(lex(program)), globalScope)