def test__recursive__lex_str_comma(): # Test lexing or and order correctly rl = RecursiveLexer() # Specify explicitly a *unicode* string for python 2 f = StringIO(u"(1,2,3,4,5)") res = rl.lex_file_obj(f) assert set(res.string_list()) >= set([',', '1', '2', '3', '4', '5'])
def test__recursive__lex_file_basic(): # Test iterating through a file object rl = RecursiveLexer() with open('test/fixtures/lexer/basic.sql') as f: res = rl.lex_file_obj(f) assert len(res) == 17 assert res[0].chunk == 'SELECT' assert res[-1].chunk == '\n'
def test__recursive__lex_str_order_not_or(): # Test lexing or and order correctly rl = RecursiveLexer() # Specify explicitly a *unicode* string for python 2 f = StringIO(u"SELECT\n a oR b as c\nFROM tbl\nORDER BY 1") res = rl.lex_file_obj(f) assert "ORDER" in res.string_list() assert "oR" in res.string_list() assert "OR" not in res.string_list()
def test__recursive__lex_file_blockcomment(): # Check we can deal with block comments in line rl = RecursiveLexer() with open('test/fixtures/lexer/block_comment.sql') as f: res = rl.lex_file_obj(f) # Check we get the block comment whole assert "/* Block comment with ending */" in res.string_list() # Check we get the field after the comment assert "a.something" in res.string_list()
def test__recursive__lex_file_inlinecomment(): # Check we can deal with block comments in line rl = RecursiveLexer() with open('test/fixtures/lexer/inline_comment.sql') as f: res = rl.lex_file_obj(f) # check that the inline comment arrives whole assert "-- This is an inline comment" in res.string_list() # The second is more tricky because it contains a quote assert "-- Sometimes they're on a new line" in res.string_list()
def test__recursive__lex_filelike(): # Test iterating through a file-like object rl = RecursiveLexer() # Specify explicitly a *unicode* string for python 2 f = StringIO(u"Select\n *\nFROM tbl\n") res = rl.lex_file_obj(f) assert res.string_list() == [ 'Select', '\n', ' ', '*', '\n', 'FROM', ' ', 'tbl', '\n' ] assert res.context_list() == [ 'content', 'whitespace', 'whitespace', 'operator', 'whitespace', 'content', 'whitespace', 'content', 'whitespace' ]