Python tokenizer Exemples

Exemple #1

Fichier : test_cparser.py Projet : peircej/python-ioLabs

def test_tokenizer():
    t=tokenizer('void* my_fn(void)')
    assert not t.empty()
    assert next(t) == 'void'
    assert next(t) == '*'
    assert next(t) == 'my_fn'
    assert next(t) == '('
    assert next(t) == 'void'
    assert next(t) == ')'
    assert t.empty()

Exemple #2

def test_tokenizer():
    t = tokenizer('void* my_fn(void)')
    assert not t.empty()
    assert next(t) == 'void'
    assert next(t) == '*'
    assert next(t) == 'my_fn'
    assert next(t) == '('
    assert next(t) == 'void'
    assert next(t) == ')'
    assert t.empty()

Exemple #3

def test_tokenizer_keywords():
    define('long long', ctypes.c_longlong)
    t = tokenizer('void* my_fn(long long)')
    assert not t.empty()
    assert next(t) == 'void'
    assert next(t) == '*'
    assert next(t) == 'my_fn'
    assert next(t) == '('
    assert next(t) == 'long long'
    assert next(t) == ')'
    assert t.empty()

Exemple #4

Fichier : test_cparser.py Projet : peircej/python-ioLabs

def test_tokenizer_keywords():
    define('long long', ctypes.c_longlong)
    t=tokenizer('void* my_fn(long long)')
    assert not t.empty()
    assert next(t) == 'void'
    assert next(t) == '*'
    assert next(t) == 'my_fn'
    assert next(t) == '('
    assert next(t) == 'long long'
    assert next(t) == ')'
    assert t.empty()