Python tokenize Exemples

Exemple #1

def test_variable_ends_text():
    assert tokenize('great is {foo}') == [{
        'type': TEXT_TOKEN,
        'value': 'great is '
    }, {
        'type': NAME_TOKEN,
        'value': 'foo'
    }]

Exemple #2

def test_open_but_no_close_after_variable():
    assert tokenize('{bar} after {foo') == [{
        'type': NAME_TOKEN,
        'value': 'bar'
    }, {
        'type': TEXT_TOKEN,
        'value': ' after {foo'
    }]

Exemple #3

def test_two_variables_follow():
    assert tokenize('{foo}{bar}') == [{
        'type': NAME_TOKEN,
        'value': 'foo'
    }, {
        'type': NAME_TOKEN,
        'value': 'bar'
    }]

Exemple #4

def test_variable_starts_text():
    assert tokenize('{foo} is great') == [{
        'type': NAME_TOKEN,
        'value': 'foo'
    }, {
        'type': TEXT_TOKEN,
        'value': ' is great'
    }]

Exemple #5

def test_whitespace_after_open_with_variable():
    assert tokenize('{ foo}{bar}') == [{
        'type': TEXT_TOKEN,
        'value': '{ foo}'
    }, {
        'type': NAME_TOKEN,
        'value': 'bar'
    }]

Exemple #6

def test_tokenize_variable_in_text():
    assert tokenize('the {foo} is great') == [{
        'type': TEXT_TOKEN,
        'value': 'the '
    }, {
        'type': NAME_TOKEN,
        'value': 'foo'
    }, {
        'type': TEXT_TOKEN,
        'value': ' is great'
    }]

Exemple #7

def test_two_variables_with_text():
    assert tokenize('a{foo}b{bar}c') == [{
        'type': TEXT_TOKEN,
        'value': 'a'
    }, {
        'type': NAME_TOKEN,
        'value': 'foo'
    }, {
        'type': TEXT_TOKEN,
        'value': 'b'
    }, {
        'type': NAME_TOKEN,
        'value': 'bar'
    }, {
        'type': TEXT_TOKEN,
        'value': 'c'
    }]

Exemple #8

def test_just_close():
    assert tokenize('foo } bar') == [{
        'type': TEXT_TOKEN,
        'value': 'foo } bar'
    }]

Exemple #9

def test_open_but_no_close_after_text():
    assert tokenize('after {foo') == [{
        'type': TEXT_TOKEN,
        'value': 'after {foo'
    }]

Exemple #10

def test_open_but_no_close():
    assert tokenize('{foo') == [{'type': TEXT_TOKEN, 'value': '{foo'}]

Exemple #11

def test_no_variables():
    assert tokenize('Hello world!') == [{
        'type': TEXT_TOKEN,
        'value': 'Hello world!'
    }]

Exemple #12

def test_tokenize_single_variable():
    assert tokenize("{foo}") == [{'type': NAME_TOKEN, 'value': 'foo'}]

Exemple #13

def test_whitespace_after_open():
    assert tokenize('{ foo}') == [{'type': TEXT_TOKEN, 'value': '{ foo}'}]

Exemple #14

def test_non_stripped_variable():
    assert tokenize('{foo }') == [{'type': NAME_TOKEN, 'value': 'foo'}]

Exemple #15

def test_whitespace_variable():
    assert tokenize('{ }') == [{'type': TEXT_TOKEN, 'value': '{ }'}]

Exemple #16

def test_empty_variable():
    assert tokenize('{}') == [{'type': TEXT_TOKEN, 'value': '{}'}]