def test__recursive__lex_str_comma():
# Test lexing or and order correctly
rl = RecursiveLexer()
# Specify explicitly a *unicode* string for python 2
f = StringIO(u"(1,2,3,4,5)")
res = rl.lex_file_obj(f)
assert set(res.string_list()) >= set([',', '1', '2', '3', '4', '5'])
def test__recursive__lex_file_basic():
# Test iterating through a file object
rl = RecursiveLexer()
with open('test/fixtures/lexer/basic.sql') as f:
res = rl.lex_file_obj(f)
assert len(res) == 17
assert res[0].chunk == 'SELECT'
assert res[-1].chunk == '\n'
def test__recursive__lex_str_order_not_or():
# Test lexing or and order correctly
rl = RecursiveLexer()
# Specify explicitly a *unicode* string for python 2
f = StringIO(u"SELECT\n a oR b as c\nFROM tbl\nORDER BY 1")
res = rl.lex_file_obj(f)
assert "ORDER" in res.string_list()
assert "oR" in res.string_list()
assert "OR" not in res.string_list()
def test__recursive__lex_file_blockcomment():
# Check we can deal with block comments in line
rl = RecursiveLexer()
with open('test/fixtures/lexer/block_comment.sql') as f:
res = rl.lex_file_obj(f)
# Check we get the block comment whole
assert "/* Block comment with ending */" in res.string_list()
# Check we get the field after the comment
assert "a.something" in res.string_list()
def test__recursive__lex_file_inlinecomment():
# Check we can deal with block comments in line
rl = RecursiveLexer()
with open('test/fixtures/lexer/inline_comment.sql') as f:
res = rl.lex_file_obj(f)
# check that the inline comment arrives whole
assert "-- This is an inline comment" in res.string_list()
# The second is more tricky because it contains a quote
assert "-- Sometimes they're on a new line" in res.string_list()
def test__recursive__lex_filelike():
# Test iterating through a file-like object
rl = RecursiveLexer()
# Specify explicitly a *unicode* string for python 2
f = StringIO(u"Select\n *\nFROM tbl\n")
res = rl.lex_file_obj(f)
assert res.string_list() == [
'Select', '\n', ' ', '*', '\n', 'FROM', ' ', 'tbl', '\n'
]
assert res.context_list() == [
'content', 'whitespace', 'whitespace', 'operator', 'whitespace',
'content', 'whitespace', 'content', 'whitespace'
]