Python tokenize Beispiele

Beispiel #1

Datei: test_charfinder.py Projekt: ZoroOP/C-and-CPP

def test_tokenize():
    assert list(tokenize('')) == []
    assert list(tokenize('a b')) == ['A', 'B']
    assert list(tokenize('a-b')) == ['A', 'B']
    assert list(tokenize('abc')) == ['ABC']
    assert list(tokenize('café')) == ['CAFÉ']

Beispiel #2

Datei: test_charfinder.py Projekt: kangheeyong/PROJECT-async-web-server

def test_tokenize():
    assert list(tokenize('')) == []
    assert list(tokenize('a b')) == ['A', 'B']
    assert list(tokenize('a-b')) == ['A', 'B']
    assert list(tokenize('abc')) == ['ABC']
    assert list(tokenize('@ab-c')) == ['AB', 'C']