Python tokenize Examples

Example #1

File: test_refine.py Project: uncletall/mwlib

def test_parse_table_cells_closed_by_next_cell():
    tokens = core.tokenize("<td>foo<td>bar")
    core.parse_table_cells(tokens, empty())
    show(tokens)
    assert tokens[0].type == T.t_complex_table_cell
    assert tokens[1].type == T.t_complex_table_cell

    assert len(tokens[0].children) == 1
    assert tokens[0].children[0]

Example #2

File: test_refine.py Project: pediapress/mwlib

def test_parse_table_cells_closed_by_next_cell():
    tokens = core.tokenize("<td>foo<td>bar")
    core.parse_table_cells(tokens, empty())
    show(tokens)
    assert tokens[0].type == T.t_complex_table_cell
    assert tokens[1].type == T.t_complex_table_cell

    assert len(tokens[0].children) == 1
    assert tokens[0].children[0]

Example #3

File: test_refine.py Project: uncletall/mwlib

def test_parse_table_cells_missing_close():
    tokens = core.tokenize("<td>bla")
    core.parse_table_cells(tokens, empty())
    show(tokens)
    assert tokens[
        0].type == T.t_complex_table_cell, "expected a complex table cell"

Example #4

File: test_refine.py Project: pediapress/mwlib

def test_parse_table_cells_missing_close():
    tokens = core.tokenize("<td>bla")
    core.parse_table_cells(tokens, empty())
    show(tokens)
    assert tokens[0].type == T.t_complex_table_cell, "expected a complex table cell"