Python Lexer.lex示例

示例#1

文件： test_lexer.py 项目： MIPS/development

    def test_lex_space(self):
        """Test whether Lexer.lex() can lex whitespaces."""

        token, end, lit = Lexer.lex(' ', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\t', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\r', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\n', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\n \r\t\n', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 5)
        self.assertEqual(lit, None)

示例#2

    def test_lex_space(self):
        """Test whether Lexer.lex() can lex whitespaces."""

        token, end, lit = Lexer.lex(' ', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\t', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\r', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\n', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\n \r\t\n', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 5)
        self.assertEqual(lit, None)

示例#3

文件： test_lexer.py 项目： MIPS/development

    def test_lex_string(self):
        """Test whether Lexer.lex() can lex a string."""

        token, end, lit = Lexer.lex('"a"', 0)
        self.assertEqual(token, Token.STRING)
        self.assertEqual(end, 3)
        self.assertEqual(lit, 'a')

        token, end, lit = Lexer.lex('`a\nb`', 0)
        self.assertEqual(token, Token.STRING)
        self.assertEqual(end, 5)
        self.assertEqual(lit, 'a\nb')

示例#4

    def test_lex_string(self):
        """Test whether Lexer.lex() can lex a string."""

        token, end, lit = Lexer.lex('"a"', 0)
        self.assertEqual(token, Token.STRING)
        self.assertEqual(end, 3)
        self.assertEqual(lit, 'a')

        token, end, lit = Lexer.lex('`a\nb`', 0)
        self.assertEqual(token, Token.STRING)
        self.assertEqual(end, 5)
        self.assertEqual(lit, 'a\nb')

示例#5

文件： test_lexer.py 项目： MIPS/development

    def test_lex_char(self):
        """Test whether Lexer.lex() can lex a character."""

        token, end, lit = Lexer.lex('(', 0)
        self.assertEqual(token, Token.LPAREN)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(')', 0)
        self.assertEqual(token, Token.RPAREN)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('[', 0)
        self.assertEqual(token, Token.LBRACKET)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(']', 0)
        self.assertEqual(token, Token.RBRACKET)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('{', 0)
        self.assertEqual(token, Token.LBRACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('}', 0)
        self.assertEqual(token, Token.RBRACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(':', 0)
        self.assertEqual(token, Token.COLON)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('=', 0)
        self.assertEqual(token, Token.ASSIGN)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('+', 0)
        self.assertEqual(token, Token.PLUS)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(',', 0)
        self.assertEqual(token, Token.COMMA)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

示例#6

    def test_lex_char(self):
        """Test whether Lexer.lex() can lex a character."""

        token, end, lit = Lexer.lex('(', 0)
        self.assertEqual(token, Token.LPAREN)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(')', 0)
        self.assertEqual(token, Token.RPAREN)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('[', 0)
        self.assertEqual(token, Token.LBRACKET)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(']', 0)
        self.assertEqual(token, Token.RBRACKET)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('{', 0)
        self.assertEqual(token, Token.LBRACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('}', 0)
        self.assertEqual(token, Token.RBRACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(':', 0)
        self.assertEqual(token, Token.COLON)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('=', 0)
        self.assertEqual(token, Token.ASSIGN)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('+', 0)
        self.assertEqual(token, Token.PLUS)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(',', 0)
        self.assertEqual(token, Token.COMMA)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

示例#7

文件： test_lexer.py 项目： MIPS/development

    def test_lex_ident(self):
        """Test whether Lexer.lex() can lex an identifier."""

        token, end, lit = Lexer.lex('ident', 0)
        self.assertEqual(token, Token.IDENT)
        self.assertEqual(end, 5)
        self.assertEqual(lit, 'ident')

示例#8

文件： test_lexer.py 项目： MIPS/development

    def test_lex_assign_plus(self):
        """Test whether Lexer.lex() can lex `+=` without problems."""

        token, end, lit = Lexer.lex('+=', 0)
        self.assertEqual(token, Token.ASSIGNPLUS)
        self.assertEqual(end, 2)
        self.assertEqual(lit, None)

示例#9

    def test_lex_ident(self):
        """Test whether Lexer.lex() can lex an identifier."""

        token, end, lit = Lexer.lex('ident', 0)
        self.assertEqual(token, Token.IDENT)
        self.assertEqual(end, 5)
        self.assertEqual(lit, 'ident')

示例#10

    def test_lex_assign_plus(self):
        """Test whether Lexer.lex() can lex `+=` without problems."""

        token, end, lit = Lexer.lex('+=', 0)
        self.assertEqual(token, Token.ASSIGNPLUS)
        self.assertEqual(end, 2)
        self.assertEqual(lit, None)

示例#11

文件： test_lexer.py 项目： MIPS/development

    def test_lex_offset(self):
        """Test the offset argument of Lexer.lex()."""

        token, end, lit = Lexer.lex('a "b"', 0)
        self.assertEqual(token, Token.IDENT)
        self.assertEqual(end, 1)
        self.assertEqual(lit, 'a')

        token, end, lit = Lexer.lex('a "b"', end)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 2)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('a "b"', end)
        self.assertEqual(token, Token.STRING)
        self.assertEqual(end, 5)
        self.assertEqual(lit, 'b')

示例#12

    def test_lex_offset(self):
        """Test the offset argument of Lexer.lex()."""

        token, end, lit = Lexer.lex('a "b"', 0)
        self.assertEqual(token, Token.IDENT)
        self.assertEqual(end, 1)
        self.assertEqual(lit, 'a')

        token, end, lit = Lexer.lex('a "b"', end)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 2)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('a "b"', end)
        self.assertEqual(token, Token.STRING)
        self.assertEqual(end, 5)
        self.assertEqual(lit, 'b')

示例#13

文件： test_lexer.py 项目： MIPS/development

    def test_lex_comment(self):
        """Test whether Lexer.lex() can lex comments."""

        token, end, lit = Lexer.lex('// abcd', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('// abcd\nnext', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a\nb*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a\n *b*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 9)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a**b*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 8)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a***b*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 9)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/**/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 4)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/***/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 5)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/**a*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 6)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a**/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 6)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/***a*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a***/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

示例#14

    def test_lex_comment(self):
        """Test whether Lexer.lex() can lex comments."""

        token, end, lit = Lexer.lex('// abcd', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('// abcd\nnext', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a\nb*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a\n *b*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 9)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a**b*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 8)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a***b*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 9)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/**/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 4)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/***/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 5)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/**a*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 6)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a**/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 6)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/***a*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a***/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)