Python tokenize示例

示例#1

def test_strip_tlp():
    assert tokenize('625-5/11-501(a)(2) (tp337049) (att)') == [
        '625', '5', '/11', '501', 'a', '2', 'att'
    ]

示例#2

def test_periods():
    assert tokenize('foo.bar') == ['foo.bar']

示例#3

def test_hyphens():
    assert tokenize('foo-bar-baz') == ['foo', 'bar', 'baz']

示例#4

def test_strip_ilcs():
    assert tokenize('720 ilcs 5/21') == ['720', '5', '/21']

示例#5

def test_commas():
    assert tokenize('foo,bar') == ['foo', 'bar']

示例#6

def test_parens():
    assert tokenize('foobar(a)') == ['foobar', 'a']
    assert tokenize('foobar(a)(b)') == ['foobar', 'a', 'b']

示例#7

def test_back_slashes():
    assert tokenize('foo\\bar') == ['foo', '/bar']

示例#8

def test_forward_slashes():
    assert tokenize('foo/bar') == ['foo', '/bar']

示例#9

def test_spaces():
    assert tokenize('foo bar') == ['foo', 'bar']
    assert tokenize('foo  bar') == ['foo', 'bar']
    assert tokenize('foo bar ') == ['foo', 'bar']
    assert tokenize(' foo bar') == ['foo', 'bar']