Python tokenize Examples

Example #1

def test_scan_simple():
    sql = "Hello 'Peter Pan' at Disney World"
    tokens = tokenize(sql)
    actual = [t[1] for t in scan(tokens)]
    expected = ["Hello", Literal("'", "Peter Pan"), "at", "Disney", "World"]
    assert actual == expected

Example #2

def test_scan_with_escape_slash():
    sql = "Hello 'Peter Pan\\\\' at Disney World"
    tokens = tokenize(sql)
    actual = [t[1] for t in scan(tokens)]
    expected = ["Hello", Literal("'", "Peter Pan\\"), "at", "Disney", "World"]
    assert actual == expected

Example #3

def test_scan_double_quotes():
    sql = """Hello 'Peter'' Pan''' at Disney World"""
    tokens = tokenize(sql)
    actual = [t[1] for t in scan(tokens)]
    expected = ["Hello", Literal("'", "Peter' Pan'"), "at", "Disney", "World"]
    assert actual == expected

Example #4

def test_scan_double_quotes_at_end():
    sql = """Hello Peter Pan at Disney 'World'"""
    tokens = tokenize(sql)
    actual = [t[1] for t in scan(tokens)]
    expected = ["Hello", "Peter", "Pan", "at", "Disney", Literal("'", "World")]
    assert actual == expected