Ejemplo n.º 1
0
    def parse(self, tokenizer, state=None):
        from rply.token import Token

        lookahead = None
        lookaheadstack = []

        statestack = [0]
        symstack = [Token("$end", "$end")]

        current_state = 0
        while True:
            if self.lr_table.default_reductions[current_state]:
                t = self.lr_table.default_reductions[current_state]
                current_state = self._reduce_production(
                    t, symstack, statestack, state
                )
                continue

            if lookahead is None:
                if lookaheadstack:
                    lookahead = lookaheadstack.pop()
                else:
                    try:
                        lookahead = next(tokenizer)
                    except StopIteration:
                        lookahead = None

                if lookahead is None:
                    lookahead = Token("$end", "$end")

            ltype = lookahead.gettokentype()
            if ltype in self.lr_table.lr_action[current_state]:
                t = self.lr_table.lr_action[current_state][ltype]
                if t > 0:
                    statestack.append(t)
                    current_state = t
                    symstack.append(lookahead)
                    lookahead = None
                    continue
                elif t < 0:
                    current_state = self._reduce_production(
                        t, symstack, statestack, state
                    )
                    continue
                else:
                    n = symstack[-1]
                    return n
            else:
                # TODO: actual error handling here
                if self.error_handler is not None:
                    if state is None:
                        self.error_handler(lookahead)
                    else:
                        self.error_handler(state, lookahead)
                    raise AssertionError("For now, error_handler must raise.")
                else:
                    raise ParsingError(None, lookahead.getsourcepos())
Ejemplo n.º 2
0
    def parse(self, tokenizer, state=None):
        from rply.token import Token

        lookahead = None
        lookaheadstack = []

        statestack = [0]
        symstack = [Token("$end", "$end")]

        current_state = 0
        while True:
            if self.lr_table.default_reductions[current_state]:
                t = self.lr_table.default_reductions[current_state]
                current_state = self._reduce_production(
                    t, symstack, statestack, state
                )
                continue

            if lookahead is None:
                if lookaheadstack:
                    lookahead = lookaheadstack.pop()
                else:
                    if tokenizer.idx < len(tokenizer.s):
                        lookahead = next(tokenizer)
                    else:
                        lookahead = None

                if lookahead is None:
                    lookahead = Token("$end", "$end")

            ltype = lookahead.gettokentype()
            if ltype in self.lr_table.lr_action[current_state]:
                t = self.lr_table.lr_action[current_state][ltype]
                if t > 0:
                    statestack.append(t)
                    current_state = t
                    symstack.append(lookahead)
                    lookahead = None
                    continue
                elif t < 0:
                    current_state = self._reduce_production(
                        t, symstack, statestack, state
                    )
                    continue
                else:
                    n = symstack[-1]
                    return n
            else:
                # TODO: actual error handling here
                if self.error_handler is not None:
                    if state is None:
                        self.error_handler(lookahead)
                    else:
                        self.error_handler(state, lookahead)
                    raise AssertionError("For now, error_handler must raise.")
                else:
                    raise ParsingError(None, lookahead.getsourcepos())
Ejemplo n.º 3
0
    def parse(self, tokenizer, state=None):
        from rply.token import Token

        lookahead = None
        lookaheadstack = []
        statestack = [0]
        symstack = [Token("$end", "$end")]
        current_state = 0
        while True:
            if self.lr_table.default_reductions[current_state]:
                t = self.lr_table.default_reductions[current_state]
                current_state = self._reduce_production(
                    t, symstack, statestack, state)
                continue
            else:
                if lookahead is None:
                    if lookaheadstack:
                        lookahead = lookaheadstack.pop()
                else:
                    try:
                        lookahead = next(tokenizer)
                    except StopIteration:
                        lookahead = None

                    if lookahead is None:
                        lookahead = Token("$end", "$end")
            ltype = lookahead.gettokentype()
            if ltype in self.lr_table.lr_action[current_state]:
                t = self.lr_table.lr_action[current_state][ltype]
                if t > 0:
                    statestack.append(t)
                    current_state = t
                    symstack.append(lookahead)
                    lookahead = None
                    continue
                else:
                    if t < 0:
                        current_state = self._reduce_production(
                            t, symstack, statestack, state)
                        continue
                    else:
                        n = symstack[(-1)]
                    return n
            elif self.error_handler is not None:
                if state is None:
                    self.error_handler(lookahead)
                else:
                    self.error_handler(state, lookahead)
                lookahead = None
                continue
            else:
                raise ParsingError(None, lookahead.getsourcepos())
Ejemplo n.º 4
0
    def parse(self, tokenizer, state=None):
        from rply.token import Token

        lookahead = None
        lookaheadstack = []

        statestack = [0]
        symstack = [Token("$end", None)]

        current_state = 0
        while True:
            if lookahead is None:
                if lookaheadstack:
                    lookahead = lookaheadstack.pop()
                else:
                    lookahead = tokenizer.next()

                if lookahead is None:
                    lookahead = Token("$end", None)

            ltype = lookahead.gettokentype()
            if ltype in self.lr_table.lr_action[current_state]:
                t = self.lr_table.lr_action[current_state][ltype]
                if t > 0:
                    statestack.append(t)
                    current_state = t
                    symstack.append(lookahead)
                    lookahead = None
                    continue
                elif t < 0:
                    # reduce a symbol on the stack and emit a production
                    p = self.lr_table.grammar.productions[-t]
                    pname = p.name
                    plen = p.getlength()
                    start = len(symstack) + (-plen - 1)
                    assert start >= 0
                    targ = symstack[start:]
                    del targ[0]
                    start = len(symstack) + (-plen)
                    assert start >= 0
                    del symstack[start:]
                    del statestack[start:]
                    if state is None:
                        value = p.func(targ)
                    else:
                        value = p.func(state, targ)
                    symstack.append(value)
                    current_state = self.lr_table.lr_goto[statestack[-1]][pname]
                    statestack.append(current_state)
                    continue
                else:
                    n = symstack[-1]
                    return n
            else:
                # TODO: actual error handling here
                if self.error_handler is not None:
                    if state is None:
                        self.error_handler(lookahead)
                    else:
                        self.error_handler(state, lookahead)
                    raise AssertionError("For now, error_handler must raise.")
                else:
                    raise ParsingError(lookahead.getsourcepos())
Ejemplo n.º 5
0
def expr_as_op(op: Token, value: ExprBox) -> SingleOpBox:
    return SingleOpBox(op=op.gettokentype(), value=value)
Ejemplo n.º 6
0
def expr_as_op(left: ExprBox, op: Token, right: ExprBox) -> OpBox:
    return OpBox(op=op.gettokentype(), left=left, right=right)
Ejemplo n.º 7
0
    def parse(self, tokenizer, state=None):
        from rply.token import Token

        lookahead = None
        lookaheadstack = []

        statestack = [0]
        symstack = [Token("$end", "$end")]

        current_state = 0
        while True:
            if self.lr_table.default_reductions[current_state]:
                t = self.lr_table.default_reductions[current_state]
                current_state = self._reduce_production(
                    t, symstack, statestack, state)
                continue

            if lookahead is None:
                if lookaheadstack:
                    lookahead = lookaheadstack.pop()
                else:
                    try:
                        # Get the next token.
                        lookahead = next(tokenizer)
                    except StopIteration:
                        lookahead = None

                if lookahead is None:
                    # Check if the only possible action from here is to end.
                    could_only_end = len(
                        self.lr_table.lr_action[current_state]) == 1
                    lookahead = Token("$end", "$end")

            ltype = lookahead.gettokentype()
            # Check if the next token is a valid next step, given our current
            # state.
            if ltype in self.lr_table.lr_action[current_state]:
                # Get the next action.
                t = self.lr_table.lr_action[current_state][ltype]
                # Shift.
                if t > 0:
                    statestack.append(t)
                    current_state = t
                    symstack.append(lookahead)
                    lookahead = None
                    continue
                # Reduce.
                elif t < 0:
                    current_state = self._reduce_production(
                        t, symstack, statestack, state)
                    continue
                # t == 0 means (maybe among other things), we got the 'end'
                # token. We are done, so we should return the token we made.
                else:
                    # This is the output token.
                    n = symstack[-1]
                    # Annotate the output token with whether or not the only
                    # next step when we got to the end, was in fact to end.
                    n._could_only_end = could_only_end
                    return n
            else:
                self.sym_stack = symstack
                self.state_stack = statestack
                self.look_ahead = lookahead
                self.look_ahead_stack = lookaheadstack
                # TODO: actual error handling here
                if self.error_handler is not None:
                    if state is None:
                        self.error_handler(lookahead)
                    else:
                        self.error_handler(state, lookahead)
                    raise AssertionError("For now, error_handler must raise.")
                else:
                    raise ParsingError(None, lookahead.getsourcepos())