Пример #1
0
def assert_continuity_of_token_spans(program_text):
    current_position = 0
    program_text_length = len(program_text)
    for token in generate_tokens(program_text):
        if token.start <= program_text_length:
            assert token.start == current_position
            current_position = token.stop
        else:
            assert token.type == name_tok["ENDMARKER"]
Пример #2
0
def test_add():
    toks = list(generate_tokens("a + b"))
    assert toks == [
        Token(0, 1, "NAME", "a"),
        Token(1, 2, "WHITESPACE", " "),
        Token(2, 3, "OP", "+"),
        Token(3, 4, "WHITESPACE", " "),
        Token(4, 5, "NAME", "b"),
        Token(6, 6, "ENDMARKER", ""),
    ]
Пример #3
0
def assert_invariant(program_text):
    tokens = generate_tokens(program_text)
    f = StringIO()
    untokenize(tokens, f)
    f.seek(0)
    assert f.read() == program_text