def test__recursive__basic_1():
rl = RecursiveLexer()
pc = PositionedChunk(' ', 0, 1, None)
res, _ = rl.lex(pc)
assert isinstance(res, ChunkString)
assert len(res) == 1
assert res[0].chunk == ' '
def test__recursive__lex_str_comma():
# Test lexing or and order correctly
rl = RecursiveLexer()
# Specify explicitly a *unicode* string for python 2
f = StringIO(u"(1,2,3,4,5)")
res = rl.lex_file_obj(f)
assert set(res.string_list()) >= set([',', '1', '2', '3', '4', '5'])
def test__recursive__multi_whitespace_a():
rl = RecursiveLexer()
pc = PositionedChunk(' SELECT \n', 0, 1, None)
res, _ = rl.lex(pc)
assert isinstance(res, ChunkString)
assert len(res) == 3
assert res[0].context == 'whitespace'
assert res[1].chunk == 'SELECT'
def test__recursive__lex_file_basic():
# Test iterating through a file object
rl = RecursiveLexer()
with open('test/fixtures/lexer/basic.sql') as f:
res = rl.lex_file_obj(f)
assert len(res) == 17
assert res[0].chunk == 'SELECT'
assert res[-1].chunk == '\n'
def test__recursive__lex_str_order_not_or():
# Test lexing or and order correctly
rl = RecursiveLexer()
# Specify explicitly a *unicode* string for python 2
f = StringIO(u"SELECT\n a oR b as c\nFROM tbl\nORDER BY 1")
res = rl.lex_file_obj(f)
assert "ORDER" in res.string_list()
assert "oR" in res.string_list()
assert "OR" not in res.string_list()
def test__recursive__lex_file_blockcomment():
# Check we can deal with block comments in line
rl = RecursiveLexer()
with open('test/fixtures/lexer/block_comment.sql') as f:
res = rl.lex_file_obj(f)
# Check we get the block comment whole
assert "/* Block comment with ending */" in res.string_list()
# Check we get the field after the comment
assert "a.something" in res.string_list()
def test__recursive__lex_file_inlinecomment():
# Check we can deal with block comments in line
rl = RecursiveLexer()
with open('test/fixtures/lexer/inline_comment.sql') as f:
res = rl.lex_file_obj(f)
# check that the inline comment arrives whole
assert "-- This is an inline comment" in res.string_list()
# The second is more tricky because it contains a quote
assert "-- Sometimes they're on a new line" in res.string_list()
def test__recursive__comment_a():
# This test requires recursion
rl = RecursiveLexer()
# The whitespace on the end of a comment should be it's own chunk
pc = PositionedChunk('SELECT -- Testing Comment\n', 0, 1, None)
res, _ = rl.lex(pc)
assert res.context_list() == [
'content', 'whitespace', 'comment', 'whitespace'
]
assert res[3].chunk == '\n'
def test__recursive__multi_whitespace_b():
# This test requires recursion
rl = RecursiveLexer()
pc = PositionedChunk(' SELECT foo \n', 0, 1, None)
res, _ = rl.lex(pc)
assert isinstance(res, ChunkString)
assert len(res) == 5
assert res[0].context == 'whitespace'
assert res[1].chunk == 'SELECT'
assert res[3].chunk == 'foo'
assert res[3].start_pos == 13
def test__recursive__lex_filelike():
# Test iterating through a file-like object
rl = RecursiveLexer()
# Specify explicitly a *unicode* string for python 2
f = StringIO(u"Select\n *\nFROM tbl\n")
res = rl.lex_file_obj(f)
assert res.string_list() == [
'Select', '\n', ' ', '*', '\n', 'FROM', ' ', 'tbl', '\n'
]
assert res.context_list() == [
'content', 'whitespace', 'whitespace', 'operator', 'whitespace',
'content', 'whitespace', 'content', 'whitespace'
]
def test__recursive__lex_chunk_buffer():
# This test requires recursion
rl = RecursiveLexer()
# The whitespace on the end of a comment should be it's own chunk
pc_list = [
PositionedChunk('SELECT\n', 0, 1, None),
PositionedChunk('NOTHING\n', 0, 2, None)
]
res, _ = rl.lex_chunk_buffer(pc_list)
assert res.context_list() == [
'content', 'whitespace', 'content', 'whitespace'
]
assert res[1].chunk == '\n'
assert res[3].chunk == '\n'