Python Lexer.lex Examples

Example #1

File: test_lexer.py Project: MIPS/development

    def test_lex_space(self):
        """Test whether Lexer.lex() can lex whitespaces."""

        token, end, lit = Lexer.lex(' ', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\t', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\r', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\n', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\n \r\t\n', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 5)
        self.assertEqual(lit, None)

Example #2

    def test_lex_space(self):
        """Test whether Lexer.lex() can lex whitespaces."""

        token, end, lit = Lexer.lex(' ', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\t', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\r', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\n', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('\n \r\t\n', 0)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 5)
        self.assertEqual(lit, None)

Example #3

File: test_lexer.py Project: MIPS/development

    def test_lex_string(self):
        """Test whether Lexer.lex() can lex a string."""

        token, end, lit = Lexer.lex('"a"', 0)
        self.assertEqual(token, Token.STRING)
        self.assertEqual(end, 3)
        self.assertEqual(lit, 'a')

        token, end, lit = Lexer.lex('`a\nb`', 0)
        self.assertEqual(token, Token.STRING)
        self.assertEqual(end, 5)
        self.assertEqual(lit, 'a\nb')

Example #4

    def test_lex_string(self):
        """Test whether Lexer.lex() can lex a string."""

        token, end, lit = Lexer.lex('"a"', 0)
        self.assertEqual(token, Token.STRING)
        self.assertEqual(end, 3)
        self.assertEqual(lit, 'a')

        token, end, lit = Lexer.lex('`a\nb`', 0)
        self.assertEqual(token, Token.STRING)
        self.assertEqual(end, 5)
        self.assertEqual(lit, 'a\nb')

Example #5

File: test_lexer.py Project: MIPS/development

    def test_lex_char(self):
        """Test whether Lexer.lex() can lex a character."""

        token, end, lit = Lexer.lex('(', 0)
        self.assertEqual(token, Token.LPAREN)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(')', 0)
        self.assertEqual(token, Token.RPAREN)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('[', 0)
        self.assertEqual(token, Token.LBRACKET)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(']', 0)
        self.assertEqual(token, Token.RBRACKET)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('{', 0)
        self.assertEqual(token, Token.LBRACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('}', 0)
        self.assertEqual(token, Token.RBRACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(':', 0)
        self.assertEqual(token, Token.COLON)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('=', 0)
        self.assertEqual(token, Token.ASSIGN)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('+', 0)
        self.assertEqual(token, Token.PLUS)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(',', 0)
        self.assertEqual(token, Token.COMMA)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

Example #6

    def test_lex_char(self):
        """Test whether Lexer.lex() can lex a character."""

        token, end, lit = Lexer.lex('(', 0)
        self.assertEqual(token, Token.LPAREN)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(')', 0)
        self.assertEqual(token, Token.RPAREN)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('[', 0)
        self.assertEqual(token, Token.LBRACKET)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(']', 0)
        self.assertEqual(token, Token.RBRACKET)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('{', 0)
        self.assertEqual(token, Token.LBRACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('}', 0)
        self.assertEqual(token, Token.RBRACE)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(':', 0)
        self.assertEqual(token, Token.COLON)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('=', 0)
        self.assertEqual(token, Token.ASSIGN)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('+', 0)
        self.assertEqual(token, Token.PLUS)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex(',', 0)
        self.assertEqual(token, Token.COMMA)
        self.assertEqual(end, 1)
        self.assertEqual(lit, None)

Example #7

File: test_lexer.py Project: MIPS/development

    def test_lex_ident(self):
        """Test whether Lexer.lex() can lex an identifier."""

        token, end, lit = Lexer.lex('ident', 0)
        self.assertEqual(token, Token.IDENT)
        self.assertEqual(end, 5)
        self.assertEqual(lit, 'ident')

Example #8

File: test_lexer.py Project: MIPS/development

    def test_lex_assign_plus(self):
        """Test whether Lexer.lex() can lex `+=` without problems."""

        token, end, lit = Lexer.lex('+=', 0)
        self.assertEqual(token, Token.ASSIGNPLUS)
        self.assertEqual(end, 2)
        self.assertEqual(lit, None)

Example #9

    def test_lex_ident(self):
        """Test whether Lexer.lex() can lex an identifier."""

        token, end, lit = Lexer.lex('ident', 0)
        self.assertEqual(token, Token.IDENT)
        self.assertEqual(end, 5)
        self.assertEqual(lit, 'ident')

Example #10

    def test_lex_assign_plus(self):
        """Test whether Lexer.lex() can lex `+=` without problems."""

        token, end, lit = Lexer.lex('+=', 0)
        self.assertEqual(token, Token.ASSIGNPLUS)
        self.assertEqual(end, 2)
        self.assertEqual(lit, None)

Example #11

File: test_lexer.py Project: MIPS/development

    def test_lex_offset(self):
        """Test the offset argument of Lexer.lex()."""

        token, end, lit = Lexer.lex('a "b"', 0)
        self.assertEqual(token, Token.IDENT)
        self.assertEqual(end, 1)
        self.assertEqual(lit, 'a')

        token, end, lit = Lexer.lex('a "b"', end)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 2)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('a "b"', end)
        self.assertEqual(token, Token.STRING)
        self.assertEqual(end, 5)
        self.assertEqual(lit, 'b')

Example #12

    def test_lex_offset(self):
        """Test the offset argument of Lexer.lex()."""

        token, end, lit = Lexer.lex('a "b"', 0)
        self.assertEqual(token, Token.IDENT)
        self.assertEqual(end, 1)
        self.assertEqual(lit, 'a')

        token, end, lit = Lexer.lex('a "b"', end)
        self.assertEqual(token, Token.SPACE)
        self.assertEqual(end, 2)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('a "b"', end)
        self.assertEqual(token, Token.STRING)
        self.assertEqual(end, 5)
        self.assertEqual(lit, 'b')

Example #13

File: test_lexer.py Project: MIPS/development

    def test_lex_comment(self):
        """Test whether Lexer.lex() can lex comments."""

        token, end, lit = Lexer.lex('// abcd', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('// abcd\nnext', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a\nb*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a\n *b*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 9)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a**b*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 8)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a***b*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 9)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/**/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 4)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/***/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 5)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/**a*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 6)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a**/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 6)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/***a*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a***/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

Example #14

    def test_lex_comment(self):
        """Test whether Lexer.lex() can lex comments."""

        token, end, lit = Lexer.lex('// abcd', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('// abcd\nnext', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a\nb*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a\n *b*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 9)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a**b*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 8)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a***b*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 9)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/**/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 4)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/***/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 5)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/**a*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 6)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a**/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 6)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/***a*/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)

        token, end, lit = Lexer.lex('/*a***/', 0)
        self.assertEqual(token, Token.COMMENT)
        self.assertEqual(end, 7)
        self.assertEqual(lit, None)