def test_pprint():
p = sqlparse.parse('select a0, b0, c0, d0, e0 from '
'(select * from dual) q0 where 1=1 and 2=2')[0]
output = StringIO()
p._pprint_tree(f=output)
pprint = '\n'.join([
"|- 0 DML 'select'", "|- 1 Whitespace ' '",
"|- 2 IdentifierList 'a0, b0...'", "| |- 0 Identifier 'a0'",
"| | `- 0 Name 'a0'", "| |- 1 Punctuation ','",
"| |- 2 Whitespace ' '", "| |- 3 Identifier 'b0'",
"| | `- 0 Name 'b0'", "| |- 4 Punctuation ','",
"| |- 5 Whitespace ' '", "| |- 6 Identifier 'c0'",
"| | `- 0 Name 'c0'", "| |- 7 Punctuation ','",
"| |- 8 Whitespace ' '", "| |- 9 Identifier 'd0'",
"| | `- 0 Name 'd0'", "| |- 10 Punctuation ','",
"| |- 11 Whitespace ' '", "| `- 12 Float 'e0'",
"|- 3 Whitespace ' '", "|- 4 Keyword 'from'", "|- 5 Whitespace ' '",
"|- 6 Identifier '(selec...'", "| |- 0 Parenthesis '(selec...'",
"| | |- 0 Punctuation '('", "| | |- 1 DML 'select'",
"| | |- 2 Whitespace ' '", "| | |- 3 Wildcard '*'",
"| | |- 4 Whitespace ' '", "| | |- 5 Keyword 'from'",
"| | |- 6 Whitespace ' '", "| | |- 7 Identifier 'dual'",
"| | | `- 0 Name 'dual'", "| | `- 8 Punctuation ')'",
"| |- 1 Whitespace ' '", "| `- 2 Identifier 'q0'",
"| `- 0 Name 'q0'", "|- 7 Whitespace ' '",
"`- 8 Where 'where ...'", " |- 0 Keyword 'where'",
" |- 1 Whitespace ' '", " |- 2 Comparison '1=1'",
" | |- 0 Integer '1'", " | |- 1 Comparison '='",
" | `- 2 Integer '1'", " |- 3 Whitespace ' '",
" |- 4 Keyword 'and'", " |- 5 Whitespace ' '",
" `- 6 Comparison '2=2'", " |- 0 Integer '2'",
" |- 1 Comparison '='", " `- 2 Integer '2'", ""
])
assert output.getvalue() == pprint
def get_tokens(self, text, unfiltered=False):
"""
Return an iterable of (tokentype, value) pairs generated from
`text`. If `unfiltered` is set to `True`, the filtering mechanism
is bypassed even if filters are defined.
Also preprocess the text, i.e. expand tabs and strip it if
wanted and applies registered filters.
"""
if isinstance(text, string_types):
if self.stripall:
text = text.strip()
elif self.stripnl:
text = text.strip('\n')
if sys.version_info[0] < 3 and isinstance(text, text_type):
text = StringIO(text.encode('utf-8'))
self.encoding = 'utf-8'
else:
text = StringIO(text)
def streamer():
for i, t, v in self.get_tokens_unprocessed(text):
yield t, v
stream = streamer()
if not unfiltered:
stream = apply_filters(stream, self.filters, self)
return stream
def get_tokens(self, text, unfiltered=False):
"""
Return an iterable of (tokentype, value) pairs generated from
`text`. If `unfiltered` is set to `True`, the filtering mechanism
is bypassed even if filters are defined.
Also preprocess the text, i.e. expand tabs and strip it if
wanted and applies registered filters.
"""
if isinstance(text, string_types):
if self.stripall:
text = text.strip()
elif self.stripnl:
text = text.strip('\n')
if sys.version_info[0] < 3 and isinstance(text, text_type):
text = StringIO(text.encode('utf-8'))
self.encoding = 'utf-8'
else:
text = StringIO(text)
def streamer():
for i, t, v in self.get_tokens_unprocessed(text):
yield t, v
stream = streamer()
if not unfiltered:
stream = apply_filters(stream, self.filters, self)
return stream
def test_pprint():
p = sqlparse.parse('select a0, b0, c0, d0, e0 from '
'(select * from dual) q0 where 1=1 and 2=2')[0]
output = StringIO()
p._pprint_tree(f=output)
pprint = '\n'.join([
" 0 DML 'select'", " 1 Whitespace ' '",
" 2 IdentifierList 'a0, b0...'", " | 0 Identifier 'a0'",
" | | 0 Name 'a0'", " | 1 Punctuation ','", " | 2 Whitespace ' '",
" | 3 Identifier 'b0'", " | | 0 Name 'b0'", " | 4 Punctuation ','",
" | 5 Whitespace ' '", " | 6 Identifier 'c0'", " | | 0 Name 'c0'",
" | 7 Punctuation ','", " | 8 Whitespace ' '",
" | 9 Identifier 'd0'", " | | 0 Name 'd0'", " | 10 Punctuation ','",
" | 11 Whitespace ' '", " | 12 Float 'e0'", " 3 Whitespace ' '",
" 4 Keyword 'from'", " 5 Whitespace ' '", " 6 Identifier '(selec...'",
" | 0 Parenthesis '(selec...'", " | | 0 Punctuation '('",
" | | 1 DML 'select'", " | | 2 Whitespace ' '",
" | | 3 Wildcard '*'", " | | 4 Whitespace ' '",
" | | 5 Keyword 'from'", " | | 6 Whitespace ' '",
" | | 7 Identifier 'dual'", " | | | 0 Name 'dual'",
" | | 8 Punctuation ')'", " | 1 Whitespace ' '",
" | 2 Identifier 'q0'", " | | 0 Name 'q0'", " 7 Whitespace ' '",
" 8 Where 'where ...'", " | 0 Keyword 'where'",
" | 1 Whitespace ' '", " | 2 Comparison '1=1'",
" | | 0 Integer '1'", " | | 1 Comparison '='",
" | | 2 Integer '1'", " | 3 Whitespace ' '", " | 4 Keyword 'and'",
" | 5 Whitespace ' '", " | 6 Comparison '2=2'",
" | | 0 Integer '2'", " | | 1 Comparison '='",
" | | 2 Integer '2'", ""
])
assert output.getvalue() == pprint
def test_stream_simple():
stream = StringIO("SELECT 1; SELECT 2;")
tokens = lexer.tokenize(stream)
assert len(list(tokens)) == 9
stream.seek(0)
tokens = list(lexer.tokenize(stream))
assert len(tokens) == 9
stream.seek(0)
tokens = list(lexer.tokenize(stream))
assert len(tokens) == 9
def test_error(self):
stream = StringIO("FOOBAR{")
lex = lexer.Lexer()
lex.bufsize = 4
tokens = list(lex.get_tokens(stream))
self.assertEqual(len(tokens), 2)
self.assertEqual(tokens[1][0], T.Error)
def get_tokens(text, encoding=None):
"""
Return an iterable of (tokentype, value) pairs generated from
`text`. If `unfiltered` is set to `True`, the filtering mechanism
is bypassed even if filters are defined.
Also preprocess the text, i.e. expand tabs and strip it if
wanted and applies registered filters.
Split ``text`` into (tokentype, text) pairs.
``stack`` is the inital stack (default: ``['root']``)
"""
encoding = encoding or 'utf-8'
if isinstance(text, string_types):
text = StringIO(text)
text = text.read()
if not isinstance(text, text_type):
try:
text = text.decode(encoding)
except UnicodeDecodeError:
text = text.decode('unicode-escape')
iterable = enumerate(text)
for pos, char in iterable:
for rexmatch, action in SQL_REGEX:
m = rexmatch(text, pos)
if not m:
continue
elif isinstance(action, tokens._TokenType):
yield action, m.group()
elif callable(action):
yield action(m.group())
consume(iterable, m.end() - pos - 1)
break
else:
yield tokens.Error, char
def get_tokens(text, encoding=None):
"""
Return an iterable of (tokentype, value) pairs generated from
`text`. If `unfiltered` is set to `True`, the filtering mechanism
is bypassed even if filters are defined.
Also preprocess the text, i.e. expand tabs and strip it if
wanted and applies registered filters.
Split ``text`` into (tokentype, text) pairs.
``stack`` is the inital stack (default: ``['root']``)
"""
encoding = encoding or 'utf-8'
if isinstance(text, string_types):
text = StringIO(text)
text = text.read()
if not isinstance(text, text_type):
try:
text = text.decode(encoding)
except UnicodeDecodeError:
text = text.decode('unicode-escape')
iterable = enumerate(text)
for pos, char in iterable:
for rexmatch, action in SQL_REGEX:
m = rexmatch(text, pos)
if not m:
continue
elif isinstance(action, tokens._TokenType):
yield action, m.group()
elif callable(action):
yield action(m.group())
consume(iterable, m.end() - pos - 1)
break
else:
yield tokens.Error, char
def ndiffAssertEqual(self, first, second):
"""Like failUnlessEqual except use ndiff for readable output."""
if first != second:
# Using the built-in .splitlines() method here will cause incorrect
# results when splitting statements that have quoted CR/CR+LF
# characters.
sfirst = split_unquoted_newlines(u(first))
ssecond = split_unquoted_newlines(u(second))
diff = difflib.ndiff(sfirst, ssecond)
fp = StringIO()
fp.write('\n')
fp.write('\n'.join(diff))
raise self.failureException(fp.getvalue())
def test_simple(self):
stream = StringIO("SELECT 1; SELECT 2;")
lex = lexer.Lexer()
tokens = lex.get_tokens(stream)
self.assertEqual(len(list(tokens)), 9)
stream.seek(0)
lex.bufsize = 4
tokens = list(lex.get_tokens(stream))
self.assertEqual(len(tokens), 9)
stream.seek(0)
lex.bufsize = len(stream.getvalue())
tokens = list(lex.get_tokens(stream))
self.assertEqual(len(tokens), 9)
def test_stream_simple():
stream = StringIO("SELECT 1; SELECT 2;")
tokens = lexer.tokenize(stream)
assert len(list(tokens)) == 9
stream.seek(0)
tokens = list(lexer.tokenize(stream))
assert len(tokens) == 9
stream.seek(0)
tokens = list(lexer.tokenize(stream))
assert len(tokens) == 9
def test_simple(self):
stream = StringIO("SELECT 1; SELECT 2;")
tokens = lexer.tokenize(stream)
self.assertEqual(len(list(tokens)), 9)
stream.seek(0)
tokens = list(lexer.tokenize(stream))
self.assertEqual(len(tokens), 9)
stream.seek(0)
tokens = list(lexer.tokenize(stream))
self.assertEqual(len(tokens), 9)
def ndiffAssertEqual(self, first, second):
"""Like failUnlessEqual except use ndiff for readable output."""
if first != second:
# Using the built-in .splitlines() method here will cause incorrect
# results when splitting statements that have quoted CR/CR+LF
# characters.
sfirst = split_unquoted_newlines(u(first))
ssecond = split_unquoted_newlines(u(second))
diff = difflib.ndiff(sfirst, ssecond)
fp = StringIO()
fp.write('\n')
fp.write('\n'.join(diff))
raise self.failureException(fp.getvalue())
def test_simple(self):
stream = StringIO("SELECT 1; SELECT 2;")
lex = lexer.Lexer()
tokens = lex.get_tokens(stream)
self.assertEqual(len(list(tokens)), 9)
stream.seek(0)
lex.bufsize = 4
tokens = list(lex.get_tokens(stream))
self.assertEqual(len(tokens), 9)
stream.seek(0)
lex.bufsize = len(stream.getvalue())
tokens = list(lex.get_tokens(stream))
self.assertEqual(len(tokens), 9)
def test_stream_error():
stream = StringIO("FOOBAR{")
tokens = list(lexer.tokenize(stream))
assert len(tokens) == 2
assert tokens[1][0] == T.Error
def test_split_stream(self):
stream = StringIO("SELECT 1; SELECT 2;")
stmts = sqlparse.parsestream(stream)
self.assertEqual(type(stmts), types.GeneratorType)
self.assertEqual(len(list(stmts)), 2)
def test_split_unicode_parsestream():
stream = StringIO(u'SELECT ö')
stmts = list(sqlparse.parsestream(stream))
assert str(stmts[0]) == 'SELECT ö'
def test_split_encoding_parsestream():
stream = StringIO("SELECT 1; SELECT 2;")
stmts = list(sqlparse.parsestream(stream))
assert isinstance(stmts[0].tokens[0].value, text_type)
def test_split_stream():
stream = StringIO("SELECT 1; SELECT 2;")
stmts = sqlparse.parsestream(stream)
assert isinstance(stmts, types.GeneratorType)
assert len(list(stmts)) == 2
def test_pprint():
p = sqlparse.parse("select a0, b0, c0, d0, e0 from " "(select * from dual) q0 where 1=1 and 2=2")[0]
output = StringIO()
p._pprint_tree(f=output)
pprint = "\n".join(
[
" 0 DML 'select'",
" 1 Whitespace ' '",
" 2 IdentifierList 'a0, b0...'",
" | 0 Identifier 'a0'",
" | | 0 Name 'a0'",
" | 1 Punctuation ','",
" | 2 Whitespace ' '",
" | 3 Identifier 'b0'",
" | | 0 Name 'b0'",
" | 4 Punctuation ','",
" | 5 Whitespace ' '",
" | 6 Identifier 'c0'",
" | | 0 Name 'c0'",
" | 7 Punctuation ','",
" | 8 Whitespace ' '",
" | 9 Identifier 'd0'",
" | | 0 Name 'd0'",
" | 10 Punctuation ','",
" | 11 Whitespace ' '",
" | 12 Float 'e0'",
" 3 Whitespace ' '",
" 4 Keyword 'from'",
" 5 Whitespace ' '",
" 6 Identifier '(selec...'",
" | 0 Parenthesis '(selec...'",
" | | 0 Punctuation '('",
" | | 1 DML 'select'",
" | | 2 Whitespace ' '",
" | | 3 Wildcard '*'",
" | | 4 Whitespace ' '",
" | | 5 Keyword 'from'",
" | | 6 Whitespace ' '",
" | | 7 Identifier 'dual'",
" | | | 0 Name 'dual'",
" | | 8 Punctuation ')'",
" | 1 Whitespace ' '",
" | 2 Identifier 'q0'",
" | | 0 Name 'q0'",
" 7 Whitespace ' '",
" 8 Where 'where ...'",
" | 0 Keyword 'where'",
" | 1 Whitespace ' '",
" | 2 Comparison '1=1'",
" | | 0 Integer '1'",
" | | 1 Comparison '='",
" | | 2 Integer '1'",
" | 3 Whitespace ' '",
" | 4 Keyword 'and'",
" | 5 Whitespace ' '",
" | 6 Comparison '2=2'",
" | | 0 Integer '2'",
" | | 1 Comparison '='",
" | | 2 Integer '2'",
"",
]
)
assert output.getvalue() == pprint
def test_encoding_parsestream(self):
stream = StringIO("SELECT 1; SELECT 2;")
stmts = list(sqlparse.parsestream(stream))
self.assertEqual(type(stmts[0].tokens[0].value), text_type)
def test_error(self):
stream = StringIO("FOOBAR{")
tokens = list(lexer.tokenize(stream))
self.assertEqual(len(tokens), 2)
self.assertEqual(tokens[1][0], T.Error)
def test_pprint():
p = sqlparse.parse('select a0, b0, c0, d0, e0 from '
'(select * from dual) q0 where 1=1 and 2=2')[0]
output = StringIO()
p._pprint_tree(f=output)
pprint = '\n'.join([
"|- 0 DML 'select'",
"|- 1 Whitespace ' '",
"|- 2 IdentifierList 'a0, b0...'",
"| |- 0 Identifier 'a0'",
"| | `- 0 Name 'a0'",
"| |- 1 Punctuation ','",
"| |- 2 Whitespace ' '",
"| |- 3 Identifier 'b0'",
"| | `- 0 Name 'b0'",
"| |- 4 Punctuation ','",
"| |- 5 Whitespace ' '",
"| |- 6 Identifier 'c0'",
"| | `- 0 Name 'c0'",
"| |- 7 Punctuation ','",
"| |- 8 Whitespace ' '",
"| |- 9 Identifier 'd0'",
"| | `- 0 Name 'd0'",
"| |- 10 Punctuation ','",
"| |- 11 Whitespace ' '",
"| `- 12 Float 'e0'",
"|- 3 Whitespace ' '",
"|- 4 Keyword 'from'",
"|- 5 Whitespace ' '",
"|- 6 Identifier '(selec...'",
"| |- 0 Parenthesis '(selec...'",
"| | |- 0 Punctuation '('",
"| | |- 1 DML 'select'",
"| | |- 2 Whitespace ' '",
"| | |- 3 Wildcard '*'",
"| | |- 4 Whitespace ' '",
"| | |- 5 Keyword 'from'",
"| | |- 6 Whitespace ' '",
"| | |- 7 Identifier 'dual'",
"| | | `- 0 Name 'dual'",
"| | `- 8 Punctuation ')'",
"| |- 1 Whitespace ' '",
"| `- 2 Identifier 'q0'",
"| `- 0 Name 'q0'",
"|- 7 Whitespace ' '",
"`- 8 Where 'where ...'",
" |- 0 Keyword 'where'",
" |- 1 Whitespace ' '",
" |- 2 Comparison '1=1'",
" | |- 0 Integer '1'",
" | |- 1 Comparison '='",
" | `- 2 Integer '1'",
" |- 3 Whitespace ' '",
" |- 4 Keyword 'and'",
" |- 5 Whitespace ' '",
" `- 6 Comparison '2=2'",
" |- 0 Integer '2'",
" |- 1 Comparison '='",
" `- 2 Integer '2'",
""])
assert output.getvalue() == pprint
