Example #1
0
    def next(self):
        while True:
            if self.idx >= len(self.s):
                raise StopIteration
            for rule in self._get_current_state().ignore_rules:
                match = rule.matches(self.s, self.idx)
                if match:
                    self._update_pos(match)
                    self._make_transition(rule)
                    break
            else:
                break

        for rule in self._get_current_state().rules:
            match = rule.matches(self.s, self.idx)
            if match:
                lineno = self._lineno
                colno = self._update_pos(match)
                source_pos = SourcePosition(match.start, lineno, colno)
                token = Token(rule.name, self.s[match.start:match.end],
                              source_pos)
                self._make_transition(rule)
                return token
        else:
            raise LexingError(None, SourcePosition(self.idx, -1, -1))
Example #2
0
    def next(self):
        while True:
            if self.idx >= len(self.s):
                raise StopIteration
            for rule in self.lexer.ignore_rules:
                match = rule.matches(self.s, self.idx)
                if match:
                    self._update_pos(match)
                    break
            else:
                break

        for rule in self.lexer.rules:
            match = rule.matches(self.s, self.idx)
            if match:
                lineno = self._lineno
                self._colno = self._update_pos(match)
                source_pos = SourcePosition(match.start, lineno, self._colno)
                if rule.name == "MISMATCH":
                    raise LexingError(
                        "%r unexpected" % self.s[match.start:match.end],
                        SourcePosition(self.idx, self._lineno, self._colno))
                token = Token(rule.name, self.s[match.start:match.end],
                              source_pos)
                return token
Example #3
0
 def 取源码位置(片段):
     if isinstance(片段, list):
         if len(片段) > 0:
             片段 = 片段[0]
     if isinstance(片段, Token):
         return 片段.getsourcepos()
     # Constant 也是 ast.expr
     if isinstance(片段, ast.stmt) or isinstance(片段, ast.expr):
         # TODO: 之前没 import SourcePosition 时, 编译/运行未报错! 需解决
         return SourcePosition(0, 片段.lineno, 片段.col_offset)
     return SourcePosition(0, 0, 0)
Example #4
0
    def next_literal_mode(self):
        # "literal" mode, i.e. outside "<?php ?>" tags: generates
        # one B_LITERAL_BLOCK until the next opening "<?php" tag
        self.mode = MODE_PHPCODE
        source = self.source
        index = self.startindex
        assert index >= 0
        tagindex = source.find('<?', index)
        if tagindex == -1:
            tagindex = len(source)
        assert tagindex >= 0
        startindex = self.startindex
        assert startindex >= 0
        block_of_text = source[startindex:tagindex]   # may be empty

        source_pos = SourcePosition(self.startindex, self.startlineno + 1, 0)
        tok = self.lexer.token_class('B_LITERAL_BLOCK', block_of_text, source_pos)
        self.startlineno += block_of_text.count('\n')
        if source[tagindex:tagindex+5].lower() == '<?php':
            pos = tagindex + 5
        elif source[tagindex:tagindex+3] == '<?=':
            pos = tagindex + 3
            self.mode = MODE_EQUALSIGN
        else:
            pos = tagindex + 2
        self.lexer.input(self.source, pos, self.startlineno)
        return tok
Example #5
0
 def next(self):
     if self.idx >= len(self.s):
         return None
     for rule in self.lexer.ignore_rules:
         match = rule.matches(self.s, self.idx)
         if match:
             self.idx = match.end
             return self.next()
     for rule in self.lexer.rules:
         match = rule.matches(self.s, self.idx)
         if match:
             # TODO: lineno and colno
             source_pos = SourcePosition(match.start, -1, -1)
             token = Token(rule.name, self.s[match.start:match.end],
                           source_pos)
             self.idx = match.end
             return token
     else:
         raise LexingError(None, SourcePosition(self.idx, -1, -1))
Example #6
0
 def next(self):
     if self.idx >= len(self.s):
         raise StopIteration
     for rule in self.lexer.ignore_rules:
         match = rule.matches(self.s, self.idx)
         if match:
             self._update_pos(match)
             return self.next()
     for rule in self.lexer.rules:
         match = rule.matches(self.s, self.idx)
         if match:
             lineno = self._lineno
             colno = self._update_pos(match)
             source_pos = SourcePosition(match.start, lineno, colno)
             token = Token(rule.name, self.s[match.start:match.end],
                           source_pos)
             return token
     else:
         raise LexingError(None, SourcePosition(self.idx, -1, -1))
Example #7
0
    def __get_position__(self, cursor):
        lineno = self.s.count("\n", 0, cursor) + 1

        colno = cursor + 1
        if lineno > 1:
            colno = colno - (self.s.rfind("\n", 0, cursor) + 1)

        sp = SourcePosition(cursor, lineno, colno)

        assert colno > 0
        assert lineno > 0
        return sp
Example #8
0
    def end_current_block(self, tok, endpos):
        # a "?>" marker that ends the current block of code
        # generates a ";" token followed by a B_LITERAL_BLOCK
        lineno = tok.source_pos.lineno
        self.startlineno = lineno
        self.startindex = endpos + 1
        self.mode = MODE_LITERAL
        if (self.startindex < len(self.source) and
                self.source[self.startindex] == '\n'):
            # self.startlineno += 1     # consume \n if immediately following
            self.startindex += 1

        return self.lexer.token_class(";", ";", SourcePosition(endpos, lineno, 0))
Example #9
0
    def next(self):
        while True:
            if self.idx >= len(self.s):
                raise StopIteration
            for rule in self.lexer.ignore_rules:
                match = rule.matches(self.s, self.idx)
                if match:
                    self._update_pos(match)
                    break
            else:
                break

        for rule in self.lexer.rules:
            match = rule.matches(self.s, self.idx)
            if match:
                lineno = self._lineno
                colno = self._update_pos(match)
                source_pos = SourcePosition(match.start, lineno, colno)
                source_str = self.s[match.start:match.end]
                name = self.lexer.reserved_dict.get(source_str, rule.name)
                token = Token(name, source_str, source_pos)
                return token
        else:
            raise LexingError(None, SourcePosition(self.idx, -1, -1))
Example #10
0
 def token(self):
     if self.pos >= len(self.buf):
         return None
     for rule, token_type in self.rules:
         m = rule.match(self.buf, pos=self.pos)
         if m:
             end = m.end()
             assert end >= 0
             val = self.buf[self.pos:end]
             tok = Token(token_type, val, SourcePosition(self.pos, self.lineno, 0))
             if token_type == "END_OF_LINE":
                 self.lineno += 1
             self.pos = end
             return tok
     raise IniLexerError(self.lineno)
Example #11
0
    def test_parse_error(self):
        pg = ParserGenerator(["VALUE"])

        @pg.production("main : VALUE")
        def main(p):
            return p[0]

        parser = pg.build()

        with py.test.raises(ParsingError) as exc_info:
            parser.parse(FakeLexer([
                Token("VALUE", "hello"),
                Token("VALUE", "world", SourcePosition(5, 10, 2)),
            ]))

        assert exc_info.value.getsourcepos().lineno == 10
Example #12
0
 def next(self):
     if self.idx >= len(self.s):
         raise StopIteration
     for rule in self.lexer.ignore_rules:
         match = rule.matches(self.s, self.idx)
         if match:
             self.idx = match.end
             return self.next()
     for rule in self.lexer.rules:
         match = rule.matches(self.s, self.idx)
         if match:
             source_pos = self.__get_position__(match.start)
             token = Token(rule.name, self.s[match.start:match.end],
                           source_pos)
             self.idx = match.end
             return token
     else:
         raise LexingError(None, SourcePosition(self.idx, -1, -1))
Example #13
0
 def whitespace_empty(self, p):
     return Token('H_WHITESPACE', '', SourcePosition(0, 0, 0))
Example #14
0
 def next_equal_sign(self):
     self.mode = MODE_PHPCODE
     source_pos = SourcePosition(self.startindex, self.startlineno + 1, 0)
     return self.lexer.token_class("T_ECHO", "echo", source_pos)
Example #15
0
 def test_repr(self):
     t = SourcePosition(1, 2, 3)
     assert repr(t) == "SourcePosition(idx=1, lineno=2, colno=3)"
Example #16
0
 def current_pos(self):
     return SourcePosition(self.idx, self.lineno, self.columno)
Example #17
0
 def test_eq(self):
     t = Token("VALUE", "3", SourcePosition(-1, -1, -1))
     assert not (t == 3)
     assert t != 3
Example #18
0
 def test_source_pos(self):
     t = Token("VALUE", "3", SourcePosition(5, 2, 1))
     assert t.getsourcepos().lineno == 2
Example #19
0
 def current_pos(self):
     """
     Return the rply SourcePosition object with current index, line number
     and column number
     """
     return SourcePosition(self.idx, self.lineno, self.columno)
Example #20
0
 def current_pos(self):
     return SourcePosition(self.get_idx(), self.get_lineno(), self.get_columno())
Example #21
0
 def __init__(self, name, source, source_pos=SourcePosition(0, 0, 0)):
     self.name = name
     self.source = source
     self.source_pos = source_pos
Example #22
0
 def test_source_pos(self):
     sp = SourcePosition(1, 2, 3)
     assert sp.idx == 1
     assert sp.lineno == 2
     assert sp.colno == 3