Example #1
0
 def testDivPunctuator(self):
     lex=Lexer()
     lex.setSrc('asd/333/=')
     lex.getToken()
     self.assertEqual(lex.getToken(), (TOK.DIV_PUNCTUATOR,'/'))
     lex.getToken()
     self.assertEqual(lex.getToken(), (TOK.DIV_PUNCTUATOR,'/='))
Example #2
0
 def testFutureStrictReservedWords(self):
     lex = Lexer()
     rw = ['implements','let','private','public','yield','interface','package','protected','static']
     lex.strictMode = True
     lex.setSrc('  '.join(rw))
     for w in rw:
         self.assertEqual(lex.getToken(), (TOK.FUTURE_RESERVED , w))
     lex.strictMode = False
     lex.setSrc(' '.join(rw))
     for w in rw:
         self.assertEqual(lex.getToken(), (TOK.ID , w))
Example #3
0
    def testCommentToken(self):
        lex = Lexer()

        lex.setSrc('//asdasd')
        self.assertEqual(lex.getToken(), (TOK.EOF, ''))

        lex.setSrc('//')
        self.assertEqual(lex.getToken(), (TOK.EOF, ''))
Example #4
0
    def testRawComment(self):
        lex = Lexer()

        lex.setSrc('//sdfsdff\n//asdasd')
        self.assertEqual(lex.getNext(), (TOK.SINGLE_COMMENT,'sdfsdff'))
        lex.getNext()
        self.assertEqual(lex.getNext(), (TOK.SINGLE_COMMENT,'asdasd'))

        lex.setSrc("""//sdfdff
        ttttt""")
        self.assertEqual(lex.getNext(), (TOK.SINGLE_COMMENT,'sdfdff'))

        lex.setSrc('/*asdasd*/')
        self.assertEqual(lex.getNext(), (TOK.MULTI_COMMENT, 'asdasd'))

        lex.setSrc('/*asd\nasd*/123123')
        self.assertEqual(lex.getNext(), (TOK.MULTINL_COMMENT, 'asd\nasd'))
Example #5
0
 def testSimpleNumber(self):
     lex = Lexer()
     lex.setSrc("3213")
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '3213'))
Example #6
0
 def testFloatNumbers(self):
     lex = Lexer()
     lex.setSrc("22.34 .232")
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '22.34'))
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '.232'))
     lex.setSrc("0. 0.3")
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '0.'))
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '0.3'))
     lex.setSrc("22. 44.")
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '22.'))
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '44.'))
Example #7
0
 def testWhitespaceSkipping(self):
     lex = Lexer()
     lex.setSrc(" \t  \v  ")
     self.assertEqual(lex.getToken(), (TOK.EOF , ''))
Example #8
0
 def testZeroNumber(self):
     lex = Lexer()
     lex.setSrc("0")
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '0'))
Example #9
0
 def __init__(self):
     self.state = 0
     self.src = ''
     self.lexer = Lexer()
Example #10
0
    def testID(self):
        lex = Lexer()

        lex.setSrc('aA$34_a bdd')
        self.assertEqual(lex.getToken(), (TOK.ID, 'aA$34_a'))
        self.assertEqual(lex.getToken(), (TOK.ID, 'bdd'))
Example #11
0
 def testNotIDStartAfterNumeric(self):
     lex = Lexer()
     lex.setSrc("0.233a")
     self.assertEqual(lex.getToken(), (TOK.ERROR , '0.233'))
     lex.setSrc("233b")
     self.assertEqual(lex.getToken(), (TOK.ERROR , '233'))
     lex.setSrc("0x233br")
     self.assertEqual(lex.getToken(), (TOK.ERROR , '0x233b'))
class Parser:
    def __init__(self):
        self.state = 0
        self.src = ''
        self.lexer = Lexer()

    def lookup(self):
        if not self.lookupToken or self.LTLookup == None:
            self.lookupToken = self.lexer.getToken(False, True)
            if self.lookupToken[0] == TOK.LT:
                self.LTLookup = True
                self.lookupToken = self.lexer.getToken()
            else:
                self.LTLookup = False
        return self.lookupToken

    def nextToken(self, REMode=False):
        if self.lookupToken != None and not REMode:
            tok = self.lookupToken
            self.lookupToken = self.LTLookup = None
            return tok
        return self.lexer.getToken(REMode)


    def reset(self):
        self.lexer.setSrc(self.src)
        self.lookupToken = None
        self.currentNode = None
        self.ASTRoot = None
        self.LTLookup = None


    def buildAST(self):
        self.reset()
        self.parseProgram()
        return self.ASTRoot

    def match(self, token, value=None):
        if value == None:
            return self.lookup()[0] == token
        else:
            return self.lookup()[0] == token and self.lookup()[1] == value

    def error(self, msg):
        pos = self.lexer.getLastTokenPos()
        raise Exception(msg + ' at line ' + str(pos['line']) + ' column ' + str(pos['column']))


    def expect(self, token, value=None):
        if not self.match(token, value):
            if value == None: value = ''
            else: value = ' ' + value
            self.error('Expected:' + JSLexer.tokenToStr(token, value) + ' ,got \'' + self.lookup()[1] + '\'')
        return self.nextToken()

    def parseProgram(self):
        self.ASTRoot = AST.ProgramNode(self.parseSourceElements())

    def parseFunctionDeclaration(self):
        self.expect(TOK.RESERVED, 'function')
        name = self.expect(TOK.ID)[1]
        arguments = []
        self.expect(TOK.PUNCTUATOR, '(')
        if self.match(TOK.ID):
            arguments.append(self.nextToken()[1])
            while self.match(TOK.PUNCTUATOR, ','):
                self.nextToken()
                arguments.append(self.expect(TOK.ID)[1])
        self.expect(TOK.PUNCTUATOR, ')')
        self.expect(TOK.PUNCTUATOR, '{')
        statements = self.parseSourceElements()
        self.expect(TOK.PUNCTUATOR, '}')

        return AST.FunctionDeclaration(name, arguments, statements)

    def parseSourceElement(self):
        if self.matchList(FIRST.FunctionDeclaration):
            return self.parseFunctionDeclaration()
        return self.parseStatement()

    def parseSourceElements(self):
        sourceElements = []
        while not self.matchList(FOLLOW.SourceElements):
            sourceElements.append(self.parseSourceElement())
        return sourceElements

    def matchList(self, list):
        token = self.lookup()
        for listToken in list:
            if type(listToken) == tuple:
                if listToken[1] == None:
                    if token[0] == listToken[0]: return True
                elif token[1] == listToken[1] and token[0] == listToken[0]:
                    return True
            else:
                if token[0] == listToken: return True
        return False

    def parseBlock(self):
        statements = []
        self.expect(TOK.PUNCTUATOR, '{')
        while not self.match(TOK.PUNCTUATOR, '}'):
            statements.append(self.parseStatement())
        self.expect(TOK.PUNCTUATOR, '}')
        return AST.Block(statements)

    def parseStatement(self):
        if self.match(TOK.PUNCTUATOR, '{'):
            return self.parseBlock()
        if self.match(TOK.RESERVED, 'var'):
            return self.parseVariableStatement()
        if self.match(TOK.PUNCTUATOR, ';'):
            self.nextToken()
            return AST.EmptyStatement()
        if self.match(TOK.RESERVED, 'if'):
            return self.parseIfStatement()
        if self.match(TOK.RESERVED, 'do'):
            return self.parseDoWhileStatement()
        if self.match(TOK.RESERVED, 'while'):
            return self.parseWhileStatement()
        if self.match(TOK.RESERVED, 'for'):
            return self.parseForStatement()
        if self.match(TOK.RESERVED, 'continue'):
            return self.parseContinueStatement()
        if self.match(TOK.RESERVED, 'break'):
            return self.parseBreakStatement()
        if self.match(TOK.RESERVED, 'return'):
            return self.parseReturnStatement()
        if self.match(TOK.RESERVED, 'with'):
            return self.parseWithStatement()
        if self.match(TOK.RESERVED, 'switch'):
            return self.parseSwitchStatement()
        if self.match(TOK.RESERVED, 'throw'):
            return self.parseThrowStatement()
        if self.match(TOK.RESERVED, 'try'):
            return self.parseTryStatement()
        if self.match(TOK.RESERVED, 'debugger'):
            self.nextToken()
            self.expectSemicolon()
            return AST.DebuggerStatement()

        return self.parseLabeledOrExpressionStatement()

    def unexpected(self):
        token = self.lookup()
        self.error('Unexpected: ' + JSLexer.tokenToStr(token))

    def parseVariableStatement(self):
        declarations = []
        self.expect(TOK.RESERVED, 'var')
        declarations = self.parseVariableDeclarationsList(False)
        if type(declarations) != list: declarations = [declarations]
        self.expectSemicolon()
        return AST.VariableStatement(declarations)

    def parseVariableDeclaration(self, noIn):
        name = self.expect(TOK.ID)[1]
        initializer = None
        if self.match(TOK.PUNCTUATOR, '='):
            self.nextToken()
            initializer = self.parseAssignmentExpression(noIn)
        return AST.VariableDeclaration(name, initializer)

    def parseAssignmentExpression(self, noIn):
        result = self.parseConditionalExpression(noIn)
        if self.matchList(AssignmentOperators):
            op = self.nextToken()[1]
            result = AST.AssignmentExpression(result, self.parseAssignmentExpression(noIn), op)
        return result

    def parseLeftHandSideExpression(self):
        # LeftHandSideExpression ::
        # (NewExpression | MemberExpression) ...
        result = None
        if self.match(TOK.RESERVED, 'new'):
            result = self.parseNewExpression()
        else:
            result = self.parseMemberExpression()

        while True:
            if self.match(TOK.PUNCTUATOR, '('):
                args = self.parseArguments()
                result = AST.Call(result, args)
            elif self.match(TOK.PUNCTUATOR, '['):
                self.nextToken()
                property = self.parseExpression(False)
                result = AST.Property(result, property)
                self.expect(TOK.PUNCTUATOR, ']')
            elif self.match(TOK.PUNCTUATOR, '.'):
                self.nextToken()
                propName = self.parseIdentifierName()
                result = AST.Property(result, propName)
            else:
                return result

    def parseNewExpression(self):
        newCount = [0]
        while self.match(TOK.RESERVED, 'new'):
            newCount[0] += 1
            self.nextToken()
        result = self.parseMemberExpression(newCount)
        while newCount[0]:
            result = AST.New(result, [])
            newCount[0] -= 1
        return result

    #we use array trick to pass mutable list and use modified value in caller
    def parseMemberExpression(self, newCount=None):
        # MemberExpression ::
        #(PrimaryExpression | FunctionLiteral)
        #   ('[' Expression ']' | '.' Identifier | Arguments)*
        result = None
        if not newCount: newCount = [0]
        if self.match(TOK.RESERVED, 'function'):
            result = self.parseFunctionExpression()
        else:
            result = self.parsePrimaryExpression()

        while True:
            if self.match(TOK.PUNCTUATOR, '('):
                if not newCount[0]: return result
                args = self.parseArguments()
                newCount[0] -= 1
                result = AST.New(result, args)
            elif self.match(TOK.PUNCTUATOR, '['):
                self.nextToken()
                property = self.parseExpression(False)
                result = AST.Property(result, property)
                self.expect(TOK.PUNCTUATOR, ']')
            elif self.match(TOK.PUNCTUATOR, '.'):
                self.nextToken()
                propName = self.parseIdentifierName()
                result = AST.Property(result, propName)
            else:
                return result


    def parsePrimaryExpression(self):
        if self.match(TOK.RESERVED, 'this'):
            self.nextToken()
            return AST.This()

        if self.match(TOK.BOOL):
            return AST.BoolLiteral(self.nextToken()[1])

        if self.match(TOK.STRING):
            return AST.Literal(self.nextToken()[1])

        if self.match(TOK.NULL):
            return AST.NullLiteral(self.nextToken()[1])

        if self.match(TOK.ID):
            token = self.nextToken()
            return AST.Identifier(token[1])

        if self.match(TOK.NUMERIC):
            token = self.nextToken()
            return AST.NumericLiteral(token[1])

        if self.match(TOK.PUNCTUATOR, '['):
            return self.parseArrayLiteral()

        if self.match(TOK.PUNCTUATOR, '{'):
            return self.parseObjectLiteral()

        if self.match(TOK.PUNCTUATOR, '('):
            self.expect(TOK.PUNCTUATOR, '(')
            result = self.parseExpression(False)
            self.expect(TOK.PUNCTUATOR, ')')
            return result

        if self.match(TOK.DIV_PUNCTUATOR):
            self.rewind()#reparse as a regexp
            return AST.RegExpLiteral(self.nextToken(True)[1])

        self.unexpected()

    def parseArguments(self):
        arguments = []
        self.expect(TOK.PUNCTUATOR, '(')
        done = self.match(TOK.PUNCTUATOR, ')')
        while not done:
            arguments.append(self.parseAssignmentExpression(False))
            if self.match(TOK.PUNCTUATOR, ','):
                self.nextToken()
            else:
                done = True
        self.expect(TOK.PUNCTUATOR, ')')
        return arguments

    def parseArrayLiteral(self):
        list = []
        self.expect(TOK.PUNCTUATOR, '[')
        done = self.match(TOK.PUNCTUATOR, ']')
        while not done:
            if self.match(TOK.PUNCTUATOR, ','):
                list.append(AST.HoleLiteral())
            else:
                list.append(self.parseAssignmentExpression(False))
            if not self.match(TOK.PUNCTUATOR, ']'):
                self.expect(TOK.PUNCTUATOR, ',')
            else:
                done = True
        self.expect(TOK.PUNCTUATOR, ']')
        return AST.Array(list)

    def parseObjectLiteral(self):
        self.expect(TOK.PUNCTUATOR, '{')
        properties = []

        while not self.match(TOK.PUNCTUATOR, '}'):
            if self.match(TOK.ID) or self.match(TOK.RESERVED) or self.match(TOK.FUTURE_RESERVED):
                key = self.nextToken()[1]
                if (key == 'get' or key == 'set') and not self.match(TOK.PUNCTUATOR, ':'):
                    #pass properties to parse function because of tricky parsing of getter-setter
                    #they must be combined to one accessor property
                    self.parseGetSetProperty(key != 'get', properties)
                    if not self.match(TOK.PUNCTUATOR, '}'): self.expect(TOK.PUNCTUATOR, ',')
                    continue
            elif self.match(TOK.NUMERIC):
                key = self.nextToken()[1]
            else:
                key = self.expect(TOK.STRING)[1]
            self.expect(TOK.PUNCTUATOR, ':')
            value = self.parseAssignmentExpression(False)

            #todo: check if accessor property exists
            properties.append(AST.ObjectProperty(key, value))

            if not self.match(TOK.PUNCTUATOR, '}'): self.expect(TOK.PUNCTUATOR, ',')

        self.expect(TOK.PUNCTUATOR, '}')
        return AST.ObjectLiteral(properties)

    def parseGetSetProperty(self, isSetter, properties):
        paramName = getterBody = setterBody = None

        if self.match(TOK.ID) or self.match(TOK.RESERVED) or self.match(TOK.FUTURE_RESERVED)\
           or self.match(TOK.NUMERIC) or self.match(TOK.STRING):
            key = self.nextToken()[1]
            self.expect(TOK.PUNCTUATOR, '(')
            if isSetter: paramName = self.expect(TOK.ID)[1]
            self.expect(TOK.PUNCTUATOR, ')')
            self.expect(TOK.PUNCTUATOR, '{')
            if isSetter: setterBody = self.parseSourceElements()
            else: getterBody = self.parseSourceElements()
            self.expect(TOK.PUNCTUATOR, '}')

            founded = None
            for s in properties:
                if s.key == key:
                    if type(s) != AST.ObjectGetSetProperty:
                        self.error('Can not have both data and accessor property with same name!')
                    else:
                        if setterBody and s.setterBody: self.error('Can not have multiple accessor property with same name!')
                        if getterBody and s.getter: self.error('Can not have multiple accessor property with same name!')
                    if isSetter:
                        s.setterBody = setterBody
                        s.paramName = paramName
                    else:
                        s.getterBody = getterBody
                    founded = s
                break
            if not founded:
                properties.append(AST.ObjectGetSetProperty(key, getterBody, setterBody, paramName))
        else:
            self.unexpected()

    def parseExpression(self, noIn):
        result = self.parseAssignmentExpression(noIn)
        while self.match(TOK.PUNCTUATOR, ','):
            self.nextToken()
            result = AST.BinaryExpression(result, self.parseAssignmentExpression(noIn), ',')
        return result

    def parseIdentifierName(self):
        if self.matchList([TOK.ID, TOK.FUTURE_RESERVED, TOK.RESERVED]):
            return AST.Identifier(self.nextToken()[1])
        self.unexpected()

    def parsePostfixExpression(self):
        result = self.parseLeftHandSideExpression()
        if not self.LTAhead() and self.matchList([(TOK.PUNCTUATOR, '++'), (TOK.PUNCTUATOR, '--')]):
            next = self.nextToken()[1]
            result = AST.PostfixExpression(result, next)
        return result

    def LTAhead(self):
        if self.LTLookup == None:
            self.lookup()
        return self.LTLookup

    def parseUnaryExpression(self):
        next = self.lookup()
        if (next[0] == TOK.PUNCTUATOR
            and (next[1] == '++' or next[1] == '--' or next[1] == '-' or next[1] == '+' or next[1] == '~' or next[1] == '!'))\
        or (next[0] == TOK.RESERVED
            and (next[1] == 'delete' or next[1] == 'typeof' or next[1] == 'void')):
            next = self.nextToken()
            return AST.UnaryExpression(self.parseUnaryExpression(), next[1])
        else:
            return self.parsePostfixExpression()

    def parseBinaryExpression(self, noIn, precedence):
        x = self.parseUnaryExpression()

        for i in reversed(range(precedence, self.Precedence(self.lookup(), noIn) + 1)):
            while self.Precedence(self.lookup(), noIn) == i:
                op = self.nextToken()[1]
                x = AST.BinaryExpression(x, self.parseBinaryExpression(noIn, i), op)

        return x

    #return -1 if not an binary op
    def Precedence(self, token, noIn):
        if token[1] == 'in' and noIn: return -1
        for prec, ops in enumerate(Precedence):
            for op in ops:
                if op == token[1]:
                    return prec
        return -1

    def rewind(self):
        self.lexer.rewind()
        self.lookupToken = self.LTLookup = None

    def parseConditionalExpression(self, noIn):
        result = self.parseBinaryExpression(noIn, 0)

        if self.match(TOK.PUNCTUATOR, '?'):
            self.nextToken()
            #left is always accept in
            left = self.parseAssignmentExpression(False)
            self.expect(TOK.PUNCTUATOR, ':')
            right = self.parseAssignmentExpression(noIn)
            result = AST.ConditionalExpression(result, left, right)

        return result

    def parseIfStatement(self):
        self.expect(TOK.RESERVED, 'if')
        self.expect(TOK.PUNCTUATOR, '(')
        condition = self.parseExpression(False)
        self.expect(TOK.PUNCTUATOR, ')')
        thenStatement = self.parseStatement()
        if self.match(TOK.RESERVED, 'else'):
            self.nextToken()
            elseStatement = self.parseStatement()
        else:
            elseStatement = AST.EmptyStatement()

        return AST.IfStatement(condition, thenStatement, elseStatement)

    def parseDoWhileStatement(self):
        self.expect(TOK.RESERVED, 'do')
        statement = self.parseStatement()
        self.expect(TOK.RESERVED, 'while')
        self.expect(TOK.PUNCTUATOR, '(')
        condition = self.parseExpression(False)
        self.expect(TOK.PUNCTUATOR, ')')
        self.expectSemicolon()
        return AST.DoWhileStatement(condition, statement)

    def parseWhileStatement(self):
        self.expect(TOK.RESERVED, 'while')
        self.expect(TOK.PUNCTUATOR, '(')
        condition = self.parseExpression(False)
        self.expect(TOK.PUNCTUATOR, ')')
        statement = self.parseStatement()
        return AST.WhileStatement(condition, statement)

    def parseForStatement(self):
        condition = next = None
        self.expect(TOK.RESERVED, 'for')
        self.expect(TOK.PUNCTUATOR, '(')
        if not self.match(TOK.PUNCTUATOR, ';'):
            if self.match(TOK.RESERVED, 'var'):
                self.nextToken()
                init = self.parseVariableDeclarationsList(True)
                if self.match(TOK.RESERVED, 'in'):
                    if type(init) == list:
                        self.error('Must be only one variable declaration in for..in statement')
                    self.nextToken()
                    enum = self.parseExpression(False)
                    self.expect(TOK.PUNCTUATOR, ')')
                    body = self.parseStatement()
                    return AST.ForInStatement(init, enum, body)
                if type(init) == list: init = AST.Block(init)
            else:
            #we parse both LeftHandSideExpression and ExpressionNoIn as an ExpressionNoIn
            #because ExpressionNoIn produces LHSE
            #additional checks may be done after that for valid LHSE if next token is 'in'
                init = self.parseExpression(True)
            if self.match(TOK.RESERVED, 'in'):
                self.nextToken()
                enum = self.parseExpression(False)
                self.expect(TOK.PUNCTUATOR, ')')
                body = self.parseStatement()
                return AST.ForInStatement(init, enum, body)
        else:
            init = AST.EmptyStatement
        self.expect(TOK.PUNCTUATOR, ';')
        if not self.match(TOK.PUNCTUATOR, ';'):
            condition = self.parseExpression(False)
        else:
            condition = AST.EmptyStatement
        self.expect(TOK.PUNCTUATOR, ';')
        if not self.match(TOK.PUNCTUATOR, ')'):
            next = self.parseExpression(False)
        else:
            next = AST.EmptyStatement
        self.expect(TOK.PUNCTUATOR, ')')
        statement = self.parseStatement()

        return AST.ForStatement(init, condition, next, statement)

    def parseVariableDeclarationsList(self, noIn):
        declarations = [self.parseVariableDeclaration(noIn)]
        while self.match(TOK.PUNCTUATOR, ','):
            self.nextToken()
            declarations.append(self.parseVariableDeclaration(noIn))
        if len(declarations) == 1: return declarations[0]
        return declarations

    def parseContinueStatement(self):
        self.expect(TOK.RESERVED, 'continue')
        label = None
        if self.LTAhead() or self.match(TOK.PUNCTUATOR, ';') or self.match(TOK.EOF):
            return AST.ContinueStatement(label)
        label = AST.Identifier(self.expect(TOK.ID)[1])
        self.expectSemicolon()
        return AST.ContinueStatement(label)

    def parseBreakStatement(self):
        self.expect(TOK.RESERVED, 'break')
        label = None
        if self.LTAhead() or self.match(TOK.PUNCTUATOR, ';') or self.match(TOK.EOF):
            return AST.BreakStatement(label)
        label = AST.Identifier(self.expect(TOK.ID)[1])
        self.expectSemicolon()
        return AST.BreakStatement(label)

    def parseReturnStatement(self):
        self.expect(TOK.RESERVED, 'return')
        result = None
        if self.LTAhead() or self.match(TOK.PUNCTUATOR, ';') or self.match(TOK.EOF):
            return AST.ReturnStatement(result)
        result = self.parseExpression(False)
        self.expectSemicolon()
        return AST.ReturnStatement(result)

    def parseWithStatement(self):
        self.expect(TOK.RESERVED, 'with')
        self.expect(TOK.PUNCTUATOR, '(')
        expr = self.parseExpression(False)
        self.expect(TOK.PUNCTUATOR, ')')
        stmt = self.parseStatement()
        return AST.WithStatement(expr, stmt)

    def parseSwitchStatement(self):
        self.expect(TOK.RESERVED, 'switch')
        self.expect(TOK.PUNCTUATOR, '(')
        expr = self.parseExpression(False)
        self.expect(TOK.PUNCTUATOR, ')')
        self.expect(TOK.PUNCTUATOR, '{')
        cases = []
        default = [False]
        while not self.match(TOK.PUNCTUATOR, '}'):
            cases.append(self.parseCaseClause(default))
        self.expect(TOK.PUNCTUATOR, '}')
        return AST.SwitchStatement(expr, cases)

    def parseCaseClause(self, default):
        label = None
        if self.match(TOK.RESERVED, 'default'):
            if default[0]: self.error('Multiple default cases not allowed')
            default[0] = True
            self.nextToken()
        else:
            self.expect(TOK.RESERVED, 'case')
            label = self.parseExpression(False)
        self.expect(TOK.PUNCTUATOR, ':')
        statements = []
        while not self.match(TOK.RESERVED, 'case') and not self.match(TOK.RESERVED, 'default') and not self.match(TOK.PUNCTUATOR, '}'):
            statements.append(self.parseStatement())
        return AST.CaseCause(label, statements)

    def expectSemicolon(self):
        if self.match(TOK.PUNCTUATOR, ';'):
            return self.nextToken()
        if self.LTAhead() or self.match(TOK.PUNCTUATOR, '}') or self.match(TOK.EOF):
            return TOK.PUNCTUATOR, ';'

        return self.expect(TOK.PUNCTUATOR, ';')

    def parseThrowStatement(self):
        self.expect(TOK.RESERVED, 'throw')
        if self.LTAhead():
            self.error('No line-terminator in throw statement allowed')
        exception = self.parseExpression(False)
        self.expectSemicolon()
        return AST.ThrowStatement(exception)

    def parseTryStatement(self):
        self.expect(TOK.RESERVED, 'try')
        block = self.parseBlock()
        catchBlock = finBlock = None
        if self.match(TOK.RESERVED,'catch'):
            catchBlock = self.parseCatchClause()
        if self.match(TOK.RESERVED,'finally'):
            finBlock = self.parseFinallyClause()

        if catchBlock == None and finBlock == None:
            self.error('try statement must have catch or finally clause')

        return AST.TryStatement(block, catchBlock, finBlock)

    def parseCatchClause(self):
        self.expect(TOK.RESERVED, 'catch')
        self.expect(TOK.PUNCTUATOR, '(')
        id = AST.Identifier(self.expect(TOK.ID)[1])
        self.expect(TOK.PUNCTUATOR, ')')
        block = self.parseBlock()
        return AST.CatchClause(id,block)

    def parseFinallyClause(self):
        self.expect(TOK.RESERVED, 'finally')
        block = self.parseBlock()
        return AST.FinallyClause(block)

    def parseLabeledOrExpressionStatement(self):
        expr = self.parseExpression(False)
        if type(expr) == AST.Identifier and self.match(TOK.PUNCTUATOR, ':'):
            self.nextToken()
            statement = self.parseStatement()
            return AST.LabelledStatement(expr, statement)
        self.expectSemicolon()
        return AST.ExpressionStatement(expr)

    def parseFunctionExpression(self):
        self.expect(TOK.RESERVED, 'function')
        name = None
        if not self.match(TOK.PUNCTUATOR, '('):
            name = self.expect(TOK.ID)[1]
        arguments = []
        self.expect(TOK.PUNCTUATOR, '(')
        if self.match(TOK.ID):
            arguments.append(self.nextToken()[1])
            while self.match(TOK.PUNCTUATOR, ','):
                self.nextToken()
                arguments.append(self.expect(TOK.ID)[1])
        self.expect(TOK.PUNCTUATOR, ')')
        self.expect(TOK.PUNCTUATOR, '{')
        statements = self.parseSourceElements()
        self.expect(TOK.PUNCTUATOR, '}')

        return AST.FunctionExpression(name, arguments, statements)
Example #13
0
 def testRegExp(self):
     lex=Lexer()
     lex.setSrc('asd/asddd/iuy')
     lex.getToken()
     self.assertEqual(lex.getToken(True), (TOK.REGEXP,'/asddd/iuy'))
     lex.setSrc('/asddd/')
     self.assertEqual(lex.getToken(True), (TOK.REGEXP,'/asddd/'))
     lex.setSrc('/\\dasddd/')
     self.assertEqual(lex.getToken(True), (TOK.REGEXP,'/\\dasddd/'))
     lex.setSrc('/[asd\\sdd](e)?:(1)+[a-z,1-9]asddd/')
     self.assertEqual(lex.getToken(True), (TOK.REGEXP,'/[asd\\sdd](e)?:(1)+[a-z,1-9]asddd/'))
Example #14
0
 def testExponentialNumbers(self):
     lex = Lexer()
     lex.setSrc("22e3 22.34e33 .232E23")
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '22e3'))
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '22.34e33'))
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '.232E23'))
     lex.setSrc("22.34e-33 .232E+23")
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '22.34e-33'))
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '.232E+23'))
     lex.setSrc("0.e+0")
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '0.e+0'))
Example #15
0
    def testStringLiteral(self):
        lex = Lexer()
        lex.setSrc('"asdasd" "ss\\"" \'\\s\'')
        self.assertEqual(lex.getToken(), (TOK.STRING, 'asdasd'))
        self.assertEqual(lex.getToken(), (TOK.STRING, 'ss"'))
        self.assertEqual(lex.getToken(), (TOK.STRING, 's'))

        lex.setSrc('"\\\\" "\\\'" "\\"" "\\b" "\\t" "\\n" "\\v" "\\f" "\\r"')

        self.assertEqual(lex.getToken(), (TOK.STRING, '\\'))
        self.assertEqual(lex.getToken(), (TOK.STRING, '\''))
        self.assertEqual(lex.getToken(), (TOK.STRING, '"'))
        self.assertEqual(lex.getToken(), (TOK.STRING, '\b'))
        self.assertEqual(lex.getToken(), (TOK.STRING, '\t'))
        self.assertEqual(lex.getToken(), (TOK.STRING, '\n'))
        self.assertEqual(lex.getToken(), (TOK.STRING, '\v'))
        self.assertEqual(lex.getToken(), (TOK.STRING, '\f'))
        self.assertEqual(lex.getToken(), (TOK.STRING, '\r'))

        lex.setSrc('"asd\\\ndd"')
        self.assertEqual(lex.getToken(), (TOK.STRING, 'asddd'))

        lex.setSrc('"dd\\u2345" \'"s\\x34\'')
        self.assertEqual(lex.getToken(), (TOK.STRING, 'dd\u2345'))
        self.assertEqual(lex.getToken(), (TOK.STRING, '"s\x34'))

        lex.setSrc('"\\x0A"')
        self.assertEqual(lex.getToken(), (TOK.STRING, '\x0A'))

        lex.setSrc("'\\0' '\\0a'")
        self.assertEqual(lex.getToken(), (TOK.STRING, '\0'))
        self.assertEqual(lex.getToken(), (TOK.STRING, '\0a'))
Example #16
0
 def testBool(self):
     lex = Lexer()
     lex.setSrc('true false')
     self.assertEqual(lex.getToken(), (TOK.BOOL, 'true'))
     self.assertEqual(lex.getToken(), (TOK.BOOL, 'false'))
Example #17
0
 def testNull(self):
     lex = Lexer()
     lex.setSrc('null null')
     self.assertEqual(lex.getToken(), (TOK.NULL, 'null'))
     self.assertEqual(lex.getToken(), (TOK.NULL, 'null'))
Example #18
0
 def testPunctuator(self):
     lex = Lexer()
     ps = ['^=','{','}','(',')','[',']','.',';',',','<','>','<=','>=','==','!=','===','!==','+','-','*','%','++','--','<<','>>','>>>','&','|','^','!','~','&&','||','?',':','=','+=','-=','*=','%=','<<=','>>=','>>>=','&=','|=','^=']
     lex.setSrc(' '.join(ps))
     for p in ps:
         self.assertEqual(lex.getToken(), (TOK.PUNCTUATOR , p))
Example #19
0
 def testFutureReservedWords(self):
     lex = Lexer()
     rw = ['class','enum','extends','super','const','export','import']
     lex.setSrc(' '.join(rw))
     for w in rw:
         self.assertEqual(lex.getToken(), (TOK.FUTURE_RESERVED , w))
Example #20
0
 def testHexNumbers(self):
     lex = Lexer()
     lex.setSrc("0x233aD")
     self.assertEqual(lex.getToken(), (TOK.NUMERIC , '0x233aD'))
Example #21
0
 def testRewind(self):
     lex = Lexer()
     lex.setSrc('asd/ee/')
     self.assertEqual(lex.getToken(),(TOK.ID, 'asd'))
     lex.rewind()
     self.assertEqual(lex.getToken(),(TOK.ID, 'asd'))
     self.assertEqual(lex.getToken(True),(TOK.REGEXP, '/ee/'))
     lex.rewind()
     self.assertEqual(lex.getToken(False),(TOK.DIV_PUNCTUATOR, '/'))
Example #22
0
 def testReservedWords(self):
     lex = Lexer()
     rw = ['break','do','instanceof','typeof','case','else','new','var','catch','finally','return','void','continue','for','switch','while','debugger','function','this','with','default','if','throw','delete','in','try']
     lex.setSrc(' '.join(rw))
     for w in rw:
         self.assertEqual(lex.getToken(), (TOK.RESERVED , w))
Example #23
0
    def testPos(self):
        lex = Lexer()
        lex.setSrc('\n123 {"asd"\n +\n/123/')

        lex.getToken()
        pos = lex.getLastTokenPos()
        self.assertEqual(pos['line'], 2)
        self.assertEqual(pos['column'], 1)

        lex.getToken()
        pos = lex.getLastTokenPos()
        self.assertEqual(pos['line'], 2)
        self.assertEqual(pos['column'], 5)

        lex.getToken()
        pos = lex.getLastTokenPos()
        self.assertEqual(pos['line'], 2)
        self.assertEqual(pos['column'], 6)

        lex.getToken()
        pos = lex.getLastTokenPos()
        self.assertEqual(pos['line'], 3)
        self.assertEqual(pos['column'], 2)

        lex.getToken()
        pos = lex.getLastTokenPos()
        self.assertEqual(pos['line'], 4)
        self.assertEqual(pos['column'], 1)
 def __init__(self):
     self.state = 0
     self.src = ''
     self.lexer = Lexer()
Example #25
0
class Parser:
    def __init__(self):
        self.state = 0
        self.src = ''
        self.lexer = Lexer()

    def lookup(self):
        if not self.lookupToken or self.LTLookup == None:
            self.lookupToken = self.lexer.getToken(False, True)
            if self.lookupToken[0] == TOK.LT:
                self.LTLookup = True
                self.lookupToken = self.lexer.getToken()
            else:
                self.LTLookup = False
        return self.lookupToken

    def nextToken(self, REMode=False):
        if self.lookupToken != None and not REMode:
            tok = self.lookupToken
            self.lookupToken = self.LTLookup = None
            return tok
        return self.lexer.getToken(REMode)

    def reset(self):
        self.lexer.setSrc(self.src)
        self.lookupToken = None
        self.currentNode = None
        self.ASTRoot = None
        self.LTLookup = None

    def buildAST(self):
        self.reset()
        self.parseProgram()
        return self.ASTRoot

    def match(self, token, value=None):
        if value == None:
            return self.lookup()[0] == token
        else:
            return self.lookup()[0] == token and self.lookup()[1] == value

    def error(self, msg):
        pos = self.lexer.getLastTokenPos()
        raise Exception(msg + ' at line ' + str(pos['line']) + ' column ' +
                        str(pos['column']))

    def expect(self, token, value=None):
        if not self.match(token, value):
            if value == None: value = ''
            else: value = ' ' + value
            self.error('Expected:' + JSLexer.tokenToStr(token, value) +
                       ' ,got \'' + self.lookup()[1] + '\'')
        return self.nextToken()

    def parseProgram(self):
        self.ASTRoot = AST.ProgramNode(self.parseSourceElements())

    def parseFunctionDeclaration(self):
        self.expect(TOK.RESERVED, 'function')
        name = self.expect(TOK.ID)[1]
        arguments = []
        self.expect(TOK.PUNCTUATOR, '(')
        if self.match(TOK.ID):
            arguments.append(self.nextToken()[1])
            while self.match(TOK.PUNCTUATOR, ','):
                self.nextToken()
                arguments.append(self.expect(TOK.ID)[1])
        self.expect(TOK.PUNCTUATOR, ')')
        self.expect(TOK.PUNCTUATOR, '{')
        statements = self.parseSourceElements()
        self.expect(TOK.PUNCTUATOR, '}')

        return AST.FunctionDeclaration(name, arguments, statements)

    def parseSourceElement(self):
        if self.matchList(FIRST.FunctionDeclaration):
            return self.parseFunctionDeclaration()
        return self.parseStatement()

    def parseSourceElements(self):
        sourceElements = []
        while not self.matchList(FOLLOW.SourceElements):
            sourceElements.append(self.parseSourceElement())
        return sourceElements

    def matchList(self, list):
        token = self.lookup()
        for listToken in list:
            if type(listToken) == tuple:
                if listToken[1] == None:
                    if token[0] == listToken[0]: return True
                elif token[1] == listToken[1] and token[0] == listToken[0]:
                    return True
            else:
                if token[0] == listToken: return True
        return False

    def parseBlock(self):
        statements = []
        self.expect(TOK.PUNCTUATOR, '{')
        while not self.match(TOK.PUNCTUATOR, '}'):
            statements.append(self.parseStatement())
        self.expect(TOK.PUNCTUATOR, '}')
        return AST.Block(statements)

    def parseStatement(self):
        if self.match(TOK.PUNCTUATOR, '{'):
            return self.parseBlock()
        if self.match(TOK.RESERVED, 'var'):
            return self.parseVariableStatement()
        if self.match(TOK.PUNCTUATOR, ';'):
            self.nextToken()
            return AST.EmptyStatement()
        if self.match(TOK.RESERVED, 'if'):
            return self.parseIfStatement()
        if self.match(TOK.RESERVED, 'do'):
            return self.parseDoWhileStatement()
        if self.match(TOK.RESERVED, 'while'):
            return self.parseWhileStatement()
        if self.match(TOK.RESERVED, 'for'):
            return self.parseForStatement()
        if self.match(TOK.RESERVED, 'continue'):
            return self.parseContinueStatement()
        if self.match(TOK.RESERVED, 'break'):
            return self.parseBreakStatement()
        if self.match(TOK.RESERVED, 'return'):
            return self.parseReturnStatement()
        if self.match(TOK.RESERVED, 'with'):
            return self.parseWithStatement()
        if self.match(TOK.RESERVED, 'switch'):
            return self.parseSwitchStatement()
        if self.match(TOK.RESERVED, 'throw'):
            return self.parseThrowStatement()
        if self.match(TOK.RESERVED, 'try'):
            return self.parseTryStatement()
        if self.match(TOK.RESERVED, 'debugger'):
            self.nextToken()
            self.expectSemicolon()
            return AST.DebuggerStatement()

        return self.parseLabeledOrExpressionStatement()

    def unexpected(self):
        token = self.lookup()
        self.error('Unexpected: ' + JSLexer.tokenToStr(token))

    def parseVariableStatement(self):
        declarations = []
        self.expect(TOK.RESERVED, 'var')
        declarations = self.parseVariableDeclarationsList(False)
        if type(declarations) != list: declarations = [declarations]
        self.expectSemicolon()
        return AST.VariableStatement(declarations)

    def parseVariableDeclaration(self, noIn):
        name = self.expect(TOK.ID)[1]
        initializer = None
        if self.match(TOK.PUNCTUATOR, '='):
            self.nextToken()
            initializer = self.parseAssignmentExpression(noIn)
        return AST.VariableDeclaration(name, initializer)

    def parseAssignmentExpression(self, noIn):
        result = self.parseConditionalExpression(noIn)
        if self.matchList(AssignmentOperators):
            op = self.nextToken()[1]
            result = AST.AssignmentExpression(
                result, self.parseAssignmentExpression(noIn), op)
        return result

    def parseLeftHandSideExpression(self):
        # LeftHandSideExpression ::
        # (NewExpression | MemberExpression) ...
        result = None
        if self.match(TOK.RESERVED, 'new'):
            result = self.parseNewExpression()
        else:
            result = self.parseMemberExpression()

        while True:
            if self.match(TOK.PUNCTUATOR, '('):
                args = self.parseArguments()
                result = AST.Call(result, args)
            elif self.match(TOK.PUNCTUATOR, '['):
                self.nextToken()
                property = self.parseExpression(False)
                result = AST.Property(result, property)
                self.expect(TOK.PUNCTUATOR, ']')
            elif self.match(TOK.PUNCTUATOR, '.'):
                self.nextToken()
                propName = self.parseIdentifierName()
                result = AST.Property(result, propName)
            else:
                return result

    def parseNewExpression(self):
        newCount = [0]
        while self.match(TOK.RESERVED, 'new'):
            newCount[0] += 1
            self.nextToken()
        result = self.parseMemberExpression(newCount)
        while newCount[0]:
            result = AST.New(result, [])
            newCount[0] -= 1
        return result

    #we use array trick to pass mutable list and use modified value in caller
    def parseMemberExpression(self, newCount=None):
        # MemberExpression ::
        #(PrimaryExpression | FunctionLiteral)
        #   ('[' Expression ']' | '.' Identifier | Arguments)*
        result = None
        if not newCount: newCount = [0]
        if self.match(TOK.RESERVED, 'function'):
            result = self.parseFunctionExpression()
        else:
            result = self.parsePrimaryExpression()

        while True:
            if self.match(TOK.PUNCTUATOR, '('):
                if not newCount[0]: return result
                args = self.parseArguments()
                newCount[0] -= 1
                result = AST.New(result, args)
            elif self.match(TOK.PUNCTUATOR, '['):
                self.nextToken()
                property = self.parseExpression(False)
                result = AST.Property(result, property)
                self.expect(TOK.PUNCTUATOR, ']')
            elif self.match(TOK.PUNCTUATOR, '.'):
                self.nextToken()
                propName = self.parseIdentifierName()
                result = AST.Property(result, propName)
            else:
                return result

    def parsePrimaryExpression(self):
        if self.match(TOK.RESERVED, 'this'):
            self.nextToken()
            return AST.This()

        if self.match(TOK.BOOL):
            return AST.BoolLiteral(self.nextToken()[1])

        if self.match(TOK.STRING):
            return AST.Literal(self.nextToken()[1])

        if self.match(TOK.NULL):
            return AST.NullLiteral(self.nextToken()[1])

        if self.match(TOK.ID):
            token = self.nextToken()
            return AST.Identifier(token[1])

        if self.match(TOK.NUMERIC):
            token = self.nextToken()
            return AST.NumericLiteral(token[1])

        if self.match(TOK.PUNCTUATOR, '['):
            return self.parseArrayLiteral()

        if self.match(TOK.PUNCTUATOR, '{'):
            return self.parseObjectLiteral()

        if self.match(TOK.PUNCTUATOR, '('):
            self.expect(TOK.PUNCTUATOR, '(')
            result = self.parseExpression(False)
            self.expect(TOK.PUNCTUATOR, ')')
            return result

        if self.match(TOK.DIV_PUNCTUATOR):
            self.rewind()  #reparse as a regexp
            return AST.RegExpLiteral(self.nextToken(True)[1])

        self.unexpected()

    def parseArguments(self):
        arguments = []
        self.expect(TOK.PUNCTUATOR, '(')
        done = self.match(TOK.PUNCTUATOR, ')')
        while not done:
            arguments.append(self.parseAssignmentExpression(False))
            if self.match(TOK.PUNCTUATOR, ','):
                self.nextToken()
            else:
                done = True
        self.expect(TOK.PUNCTUATOR, ')')
        return arguments

    def parseArrayLiteral(self):
        list = []
        self.expect(TOK.PUNCTUATOR, '[')
        done = self.match(TOK.PUNCTUATOR, ']')
        while not done:
            if self.match(TOK.PUNCTUATOR, ','):
                list.append(AST.HoleLiteral())
            else:
                list.append(self.parseAssignmentExpression(False))
            if not self.match(TOK.PUNCTUATOR, ']'):
                self.expect(TOK.PUNCTUATOR, ',')
            else:
                done = True
        self.expect(TOK.PUNCTUATOR, ']')
        return AST.Array(list)

    def parseObjectLiteral(self):
        self.expect(TOK.PUNCTUATOR, '{')
        properties = []

        while not self.match(TOK.PUNCTUATOR, '}'):
            if self.match(TOK.ID) or self.match(TOK.RESERVED) or self.match(
                    TOK.FUTURE_RESERVED):
                key = self.nextToken()[1]
                if (key == 'get' or key
                        == 'set') and not self.match(TOK.PUNCTUATOR, ':'):
                    #pass properties to parse function because of tricky parsing of getter-setter
                    #they must be combined to one accessor property
                    self.parseGetSetProperty(key != 'get', properties)
                    if not self.match(TOK.PUNCTUATOR, '}'):
                        self.expect(TOK.PUNCTUATOR, ',')
                    continue
            elif self.match(TOK.NUMERIC):
                key = self.nextToken()[1]
            else:
                key = self.expect(TOK.STRING)[1]
            self.expect(TOK.PUNCTUATOR, ':')
            value = self.parseAssignmentExpression(False)

            #todo: check if accessor property exists
            properties.append(AST.ObjectProperty(key, value))

            if not self.match(TOK.PUNCTUATOR, '}'):
                self.expect(TOK.PUNCTUATOR, ',')

        self.expect(TOK.PUNCTUATOR, '}')
        return AST.ObjectLiteral(properties)

    def parseGetSetProperty(self, isSetter, properties):
        paramName = getterBody = setterBody = None

        if self.match(TOK.ID) or self.match(TOK.RESERVED) or self.match(TOK.FUTURE_RESERVED)\
           or self.match(TOK.NUMERIC) or self.match(TOK.STRING):
            key = self.nextToken()[1]
            self.expect(TOK.PUNCTUATOR, '(')
            if isSetter: paramName = self.expect(TOK.ID)[1]
            self.expect(TOK.PUNCTUATOR, ')')
            self.expect(TOK.PUNCTUATOR, '{')
            if isSetter: setterBody = self.parseSourceElements()
            else: getterBody = self.parseSourceElements()
            self.expect(TOK.PUNCTUATOR, '}')

            founded = None
            for s in properties:
                if s.key == key:
                    if type(s) != AST.ObjectGetSetProperty:
                        self.error(
                            'Can not have both data and accessor property with same name!'
                        )
                    else:
                        if setterBody and s.setterBody:
                            self.error(
                                'Can not have multiple accessor property with same name!'
                            )
                        if getterBody and s.getter:
                            self.error(
                                'Can not have multiple accessor property with same name!'
                            )
                    if isSetter:
                        s.setterBody = setterBody
                        s.paramName = paramName
                    else:
                        s.getterBody = getterBody
                    founded = s
                break
            if not founded:
                properties.append(
                    AST.ObjectGetSetProperty(key, getterBody, setterBody,
                                             paramName))
        else:
            self.unexpected()

    def parseExpression(self, noIn):
        result = self.parseAssignmentExpression(noIn)
        while self.match(TOK.PUNCTUATOR, ','):
            self.nextToken()
            result = AST.BinaryExpression(result,
                                          self.parseAssignmentExpression(noIn),
                                          ',')
        return result

    def parseIdentifierName(self):
        if self.matchList([TOK.ID, TOK.FUTURE_RESERVED, TOK.RESERVED]):
            return AST.Identifier(self.nextToken()[1])
        self.unexpected()

    def parsePostfixExpression(self):
        result = self.parseLeftHandSideExpression()
        if not self.LTAhead() and self.matchList([(TOK.PUNCTUATOR, '++'),
                                                  (TOK.PUNCTUATOR, '--')]):
            next = self.nextToken()[1]
            result = AST.PostfixExpression(result, next)
        return result

    def LTAhead(self):
        if self.LTLookup == None:
            self.lookup()
        return self.LTLookup

    def parseUnaryExpression(self):
        next = self.lookup()
        if (next[0] == TOK.PUNCTUATOR
            and (next[1] == '++' or next[1] == '--' or next[1] == '-' or next[1] == '+' or next[1] == '~' or next[1] == '!'))\
        or (next[0] == TOK.RESERVED
            and (next[1] == 'delete' or next[1] == 'typeof' or next[1] == 'void')):
            next = self.nextToken()
            return AST.UnaryExpression(self.parseUnaryExpression(), next[1])
        else:
            return self.parsePostfixExpression()

    def parseBinaryExpression(self, noIn, precedence):
        x = self.parseUnaryExpression()

        for i in reversed(
                range(precedence,
                      self.Precedence(self.lookup(), noIn) + 1)):
            while self.Precedence(self.lookup(), noIn) == i:
                op = self.nextToken()[1]
                x = AST.BinaryExpression(x,
                                         self.parseBinaryExpression(noIn,
                                                                    i), op)

        return x

    #return -1 if not an binary op
    def Precedence(self, token, noIn):
        if token[1] == 'in' and noIn: return -1
        for prec, ops in enumerate(Precedence):
            for op in ops:
                if op == token[1]:
                    return prec
        return -1

    def rewind(self):
        self.lexer.rewind()
        self.lookupToken = self.LTLookup = None

    def parseConditionalExpression(self, noIn):
        result = self.parseBinaryExpression(noIn, 0)

        if self.match(TOK.PUNCTUATOR, '?'):
            self.nextToken()
            #left is always accept in
            left = self.parseAssignmentExpression(False)
            self.expect(TOK.PUNCTUATOR, ':')
            right = self.parseAssignmentExpression(noIn)
            result = AST.ConditionalExpression(result, left, right)

        return result

    def parseIfStatement(self):
        self.expect(TOK.RESERVED, 'if')
        self.expect(TOK.PUNCTUATOR, '(')
        condition = self.parseExpression(False)
        self.expect(TOK.PUNCTUATOR, ')')
        thenStatement = self.parseStatement()
        if self.match(TOK.RESERVED, 'else'):
            self.nextToken()
            elseStatement = self.parseStatement()
        else:
            elseStatement = AST.EmptyStatement()

        return AST.IfStatement(condition, thenStatement, elseStatement)

    def parseDoWhileStatement(self):
        self.expect(TOK.RESERVED, 'do')
        statement = self.parseStatement()
        self.expect(TOK.RESERVED, 'while')
        self.expect(TOK.PUNCTUATOR, '(')
        condition = self.parseExpression(False)
        self.expect(TOK.PUNCTUATOR, ')')
        self.expectSemicolon()
        return AST.DoWhileStatement(condition, statement)

    def parseWhileStatement(self):
        self.expect(TOK.RESERVED, 'while')
        self.expect(TOK.PUNCTUATOR, '(')
        condition = self.parseExpression(False)
        self.expect(TOK.PUNCTUATOR, ')')
        statement = self.parseStatement()
        return AST.WhileStatement(condition, statement)

    def parseForStatement(self):
        condition = next = None
        self.expect(TOK.RESERVED, 'for')
        self.expect(TOK.PUNCTUATOR, '(')
        if not self.match(TOK.PUNCTUATOR, ';'):
            if self.match(TOK.RESERVED, 'var'):
                self.nextToken()
                init = self.parseVariableDeclarationsList(True)
                if self.match(TOK.RESERVED, 'in'):
                    if type(init) == list:
                        self.error(
                            'Must be only one variable declaration in for..in statement'
                        )
                    self.nextToken()
                    enum = self.parseExpression(False)
                    self.expect(TOK.PUNCTUATOR, ')')
                    body = self.parseStatement()
                    return AST.ForInStatement(init, enum, body)
                if type(init) == list: init = AST.Block(init)
            else:
                #we parse both LeftHandSideExpression and ExpressionNoIn as an ExpressionNoIn
                #because ExpressionNoIn produces LHSE
                #additional checks may be done after that for valid LHSE if next token is 'in'
                init = self.parseExpression(True)
            if self.match(TOK.RESERVED, 'in'):
                self.nextToken()
                enum = self.parseExpression(False)
                self.expect(TOK.PUNCTUATOR, ')')
                body = self.parseStatement()
                return AST.ForInStatement(init, enum, body)
        else:
            init = AST.EmptyStatement
        self.expect(TOK.PUNCTUATOR, ';')
        if not self.match(TOK.PUNCTUATOR, ';'):
            condition = self.parseExpression(False)
        else:
            condition = AST.EmptyStatement
        self.expect(TOK.PUNCTUATOR, ';')
        if not self.match(TOK.PUNCTUATOR, ')'):
            next = self.parseExpression(False)
        else:
            next = AST.EmptyStatement
        self.expect(TOK.PUNCTUATOR, ')')
        statement = self.parseStatement()

        return AST.ForStatement(init, condition, next, statement)

    def parseVariableDeclarationsList(self, noIn):
        declarations = [self.parseVariableDeclaration(noIn)]
        while self.match(TOK.PUNCTUATOR, ','):
            self.nextToken()
            declarations.append(self.parseVariableDeclaration(noIn))
        if len(declarations) == 1: return declarations[0]
        return declarations

    def parseContinueStatement(self):
        self.expect(TOK.RESERVED, 'continue')
        label = None
        if self.LTAhead() or self.match(TOK.PUNCTUATOR, ';') or self.match(
                TOK.EOF):
            return AST.ContinueStatement(label)
        label = AST.Identifier(self.expect(TOK.ID)[1])
        self.expectSemicolon()
        return AST.ContinueStatement(label)

    def parseBreakStatement(self):
        self.expect(TOK.RESERVED, 'break')
        label = None
        if self.LTAhead() or self.match(TOK.PUNCTUATOR, ';') or self.match(
                TOK.EOF):
            return AST.BreakStatement(label)
        label = AST.Identifier(self.expect(TOK.ID)[1])
        self.expectSemicolon()
        return AST.BreakStatement(label)

    def parseReturnStatement(self):
        self.expect(TOK.RESERVED, 'return')
        result = None
        if self.LTAhead() or self.match(TOK.PUNCTUATOR, ';') or self.match(
                TOK.EOF):
            return AST.ReturnStatement(result)
        result = self.parseExpression(False)
        self.expectSemicolon()
        return AST.ReturnStatement(result)

    def parseWithStatement(self):
        self.expect(TOK.RESERVED, 'with')
        self.expect(TOK.PUNCTUATOR, '(')
        expr = self.parseExpression(False)
        self.expect(TOK.PUNCTUATOR, ')')
        stmt = self.parseStatement()
        return AST.WithStatement(expr, stmt)

    def parseSwitchStatement(self):
        self.expect(TOK.RESERVED, 'switch')
        self.expect(TOK.PUNCTUATOR, '(')
        expr = self.parseExpression(False)
        self.expect(TOK.PUNCTUATOR, ')')
        self.expect(TOK.PUNCTUATOR, '{')
        cases = []
        default = [False]
        while not self.match(TOK.PUNCTUATOR, '}'):
            cases.append(self.parseCaseClause(default))
        self.expect(TOK.PUNCTUATOR, '}')
        return AST.SwitchStatement(expr, cases)

    def parseCaseClause(self, default):
        label = None
        if self.match(TOK.RESERVED, 'default'):
            if default[0]: self.error('Multiple default cases not allowed')
            default[0] = True
            self.nextToken()
        else:
            self.expect(TOK.RESERVED, 'case')
            label = self.parseExpression(False)
        self.expect(TOK.PUNCTUATOR, ':')
        statements = []
        while not self.match(TOK.RESERVED, 'case') and not self.match(
                TOK.RESERVED, 'default') and not self.match(
                    TOK.PUNCTUATOR, '}'):
            statements.append(self.parseStatement())
        return AST.CaseCause(label, statements)

    def expectSemicolon(self):
        if self.match(TOK.PUNCTUATOR, ';'):
            return self.nextToken()
        if self.LTAhead() or self.match(TOK.PUNCTUATOR, '}') or self.match(
                TOK.EOF):
            return TOK.PUNCTUATOR, ';'

        return self.expect(TOK.PUNCTUATOR, ';')

    def parseThrowStatement(self):
        self.expect(TOK.RESERVED, 'throw')
        if self.LTAhead():
            self.error('No line-terminator in throw statement allowed')
        exception = self.parseExpression(False)
        self.expectSemicolon()
        return AST.ThrowStatement(exception)

    def parseTryStatement(self):
        self.expect(TOK.RESERVED, 'try')
        block = self.parseBlock()
        catchBlock = finBlock = None
        if self.match(TOK.RESERVED, 'catch'):
            catchBlock = self.parseCatchClause()
        if self.match(TOK.RESERVED, 'finally'):
            finBlock = self.parseFinallyClause()

        if catchBlock == None and finBlock == None:
            self.error('try statement must have catch or finally clause')

        return AST.TryStatement(block, catchBlock, finBlock)

    def parseCatchClause(self):
        self.expect(TOK.RESERVED, 'catch')
        self.expect(TOK.PUNCTUATOR, '(')
        id = AST.Identifier(self.expect(TOK.ID)[1])
        self.expect(TOK.PUNCTUATOR, ')')
        block = self.parseBlock()
        return AST.CatchClause(id, block)

    def parseFinallyClause(self):
        self.expect(TOK.RESERVED, 'finally')
        block = self.parseBlock()
        return AST.FinallyClause(block)

    def parseLabeledOrExpressionStatement(self):
        expr = self.parseExpression(False)
        if type(expr) == AST.Identifier and self.match(TOK.PUNCTUATOR, ':'):
            self.nextToken()
            statement = self.parseStatement()
            return AST.LabelledStatement(expr, statement)
        self.expectSemicolon()
        return AST.ExpressionStatement(expr)

    def parseFunctionExpression(self):
        self.expect(TOK.RESERVED, 'function')
        name = None
        if not self.match(TOK.PUNCTUATOR, '('):
            name = self.expect(TOK.ID)[1]
        arguments = []
        self.expect(TOK.PUNCTUATOR, '(')
        if self.match(TOK.ID):
            arguments.append(self.nextToken()[1])
            while self.match(TOK.PUNCTUATOR, ','):
                self.nextToken()
                arguments.append(self.expect(TOK.ID)[1])
        self.expect(TOK.PUNCTUATOR, ')')
        self.expect(TOK.PUNCTUATOR, '{')
        statements = self.parseSourceElements()
        self.expect(TOK.PUNCTUATOR, '}')

        return AST.FunctionExpression(name, arguments, statements)