Python Lexer Examples

Example #1

File: test_lexer.py Project: aroberge/lsbasi

def test_tokens_part9():
    """New tokens from part 9 or the tutorial"""
    records = (
        (':=', my_spi.ASSIGN, ':='),
        ('.', my_spi.DOT, '.'),
        ('number', my_spi.ID, 'number'),
        (';', my_spi.SEMI, ';'),
        ('BEGIN', my_spi.BEGIN, 'BEGIN'),
        ('END', my_spi.END, 'END'),
    )
    for text, tok_type, tok_val in records:
        lexer = my_spi.Lexer(text)
        token = lexer.get_next_token()
        assert token.type == tok_type
        assert token.value == tok_val

Example #2

File: test_lexer.py Project: aroberge/lsbasi

def test_lexer_subtraction():
    lexer = my_spi.Lexer(" 32 - 3  ")

    token = lexer.get_next_token()
    assert token.type == my_spi.INTEGER
    assert token.value == 32

    token = lexer.get_next_token()
    assert token.type == my_spi.MINUS
    assert token.value == "-"

    token = lexer.get_next_token()
    assert token.type == my_spi.INTEGER
    assert token.value == 3

    token = lexer.get_next_token()
    assert token.type == my_spi.EOF
    assert token.value is None

Example #3

File: test_lexer.py Project: aroberge/lsbasi

def test_lexer_addition():
    lexer = my_spi.Lexer("2+3")

    token = lexer.get_next_token()
    assert token.type == my_spi.INTEGER
    assert token.value == 2

    token = lexer.get_next_token()
    assert token.type == my_spi.PLUS
    assert token.value == "+"

    token = lexer.get_next_token()
    assert token.type == my_spi.INTEGER
    assert token.value == 3

    token = lexer.get_next_token()
    assert token.type == my_spi.EOF
    assert token.value is None

Example #4

File: test_lexer.py Project: aroberge/lsbasi

def test_lexer_multidigit_integer():
    lexer = my_spi.Lexer("234")
    token = lexer.get_next_token()
    assert token.type == my_spi.INTEGER
    assert token.value == 234

Example #5

File: test_lexer.py Project: aroberge/lsbasi

def test_lexer_whitespace():
    lexer = my_spi.Lexer("  23")
    token = lexer.get_next_token()
    assert token.type == my_spi.INTEGER
    assert token.value == 23

Example #6

File: test_lexer.py Project: aroberge/lsbasi

def test_lexer_right_paren():
    lexer = my_spi.Lexer(")")
    token = lexer.get_next_token()
    assert token.type == my_spi.RPAREN
    assert token.value == ")"

Example #7

File: test_lexer.py Project: aroberge/lsbasi

def test_lexer_eof():
    lexer = my_spi.Lexer("-")
    token = lexer.get_next_token()
    token = lexer.get_next_token()
    assert token.type == my_spi.EOF

Example #8

File: test_lexer.py Project: aroberge/lsbasi

def test_lexer_minus():
    lexer = my_spi.Lexer("-")
    token = lexer.get_next_token()
    assert token.type == my_spi.MINUS
    assert token.value == "-"

Example #9

File: test_lexer.py Project: aroberge/lsbasi

def test_lexer_plus():
    lexer = my_spi.Lexer("+")
    token = lexer.get_next_token()
    assert token.type == my_spi.PLUS
    assert token.value == "+"

Example #10

File: test_my_spi.py Project: aroberge/lsbasi

def make_interpreter(text):
    lexer = my_spi.Lexer(text)
    parser = my_spi.Parser(lexer)
    return my_spi.Interpreter(parser)

Example #11

File: test_my_calc.py Project: aroberge/lsbasi

def calc(text):
    lexer = my_spi.Lexer(text)
    parser = my_spi.Parser(lexer)
    interpreter = my_spi.Interpreter(parser)
    return interpreter.calc()