Python tokenizer示例

示例#1

文件： test_cparser.py 项目： peircej/python-ioLabs

def test_tokenizer():
    t=tokenizer('void* my_fn(void)')
    assert not t.empty()
    assert next(t) == 'void'
    assert next(t) == '*'
    assert next(t) == 'my_fn'
    assert next(t) == '('
    assert next(t) == 'void'
    assert next(t) == ')'
    assert t.empty()

示例#2

def test_tokenizer():
    t = tokenizer('void* my_fn(void)')
    assert not t.empty()
    assert next(t) == 'void'
    assert next(t) == '*'
    assert next(t) == 'my_fn'
    assert next(t) == '('
    assert next(t) == 'void'
    assert next(t) == ')'
    assert t.empty()

示例#3

def test_tokenizer_keywords():
    define('long long', ctypes.c_longlong)
    t = tokenizer('void* my_fn(long long)')
    assert not t.empty()
    assert next(t) == 'void'
    assert next(t) == '*'
    assert next(t) == 'my_fn'
    assert next(t) == '('
    assert next(t) == 'long long'
    assert next(t) == ')'
    assert t.empty()

示例#4

文件： test_cparser.py 项目： peircej/python-ioLabs

def test_tokenizer_keywords():
    define('long long', ctypes.c_longlong)
    t=tokenizer('void* my_fn(long long)')
    assert not t.empty()
    assert next(t) == 'void'
    assert next(t) == '*'
    assert next(t) == 'my_fn'
    assert next(t) == '('
    assert next(t) == 'long long'
    assert next(t) == ')'
    assert t.empty()