def ensure_sqlparse(
sql: Union[sqlparse.sql.Statement, str]) -> sqlparse.sql.Statement:
"""Returns a :class:`sqlparse.sql.Statement` from ``sql``.
Will return ``sql`` with no modification if it's already a
:mod:`sqlparse.sql` object.
Needed to accommodate dynamic addition of statements as strings to
an existing :class:`~snowmobile.Script` object from
from raw strings as opposed to a :class:`sqlparse.sql.Statement`
objects as is done when reading a sql file.
Args:
sql (Union[sqlparse.sql.Statement, str]):
Either a string of sql or an already parsed
sqlparse.sql.Statement object.
Returns (sqlparse.sql.Statement):
A parsed sql statement.
"""
if isinstance(sql, sqlparse.sql.Statement):
return sql
parsed = [
s for s in sqlparse.parsestream(stream=sql.strip().strip(";"))
if not s.value.isspace()
]
assert (
parsed
), f'sqlparse.parsestream("""\n{sql}"""\n) returned an empty statement.'
return parsed[0]
def test_split_stream(self):
import types
from cStringIO import StringIO
stream = StringIO("SELECT 1; SELECT 2;")
stmts = sqlparse.parsestream(stream)
self.assertEqual(type(stmts), types.GeneratorType)
self.assertEqual(len(list(stmts)), 2)
def test_split_stream(self):
import types
from cStringIO import StringIO
stream = StringIO("SELECT 1; SELECT 2;")
stmts = sqlparse.parsestream(stream)
self.assertEqual(type(stmts), types.GeneratorType)
self.assertEqual(len(list(stmts)), 2)
def execute(engine, file_path):
with open(file_path) as f:
for parsed_statement in parsestream(f):
statement = stringtype(parsed_statement).strip()
if statement == '':
continue
if log is not None:
log.info("Running: {statement}".format(statement=statement))
try:
engine.execute(statement)
except ProgrammingError as e:
if log is not None:
log.warning(e.message)
else:
raise e
def _get_statements(path):
'''
Get statements from file
'''
with codecs.open(path, encoding='utf-8') as i:
data = i.read()
if u'/* pgmigrate-encoding: utf-8 */' not in data:
try:
data.encode('ascii')
except UnicodeError as exc:
raise MalformedStatement(
'Non ascii symbols in file: {0}, {1}'.format(path, text(exc)))
for statement in sqlparse.parsestream(data, encoding='utf-8'):
st_str = text(statement).strip().encode('utf-8')
if st_str:
yield st_str
def _get_statements(path):
"""
Get statements from file
"""
with codecs.open(path, encoding='utf-8') as i:
data = i.read()
if MIGRATION_DEFAULT_ENCODING not in data:
try:
data.encode('ascii')
except UnicodeError as exc:
raise MalformedStatement(
'Non ascii symbols in file: {0}, {1}'.format(path, text(exc)))
data = sqlparse.format(data, strip_comments=True)
for statement in sqlparse.parsestream(data, encoding='utf-8'):
st_str = text(statement).strip().encode('utf-8')
if st_str:
yield st_str
def test_split_unicode_parsestream():
stream = StringIO(u'SELECT ö')
stmts = list(sqlparse.parsestream(stream))
assert str(stmts[0]) == 'SELECT ö'
def test_split_encoding_parsestream():
stream = StringIO("SELECT 1; SELECT 2;")
stmts = list(sqlparse.parsestream(stream))
assert isinstance(stmts[0].tokens[0].value, text_type)
def test_split_stream():
stream = StringIO("SELECT 1; SELECT 2;")
stmts = sqlparse.parsestream(stream)
assert isinstance(stmts, types.GeneratorType)
assert len(list(stmts)) == 2
def test_split_unicode_parsestream():
stream = StringIO(u'SELECT ö')
stmts = list(sqlparse.parsestream(stream))
assert str(stmts[0]) == 'SELECT ö'
def test_split_encoding_parsestream():
stream = StringIO("SELECT 1; SELECT 2;")
stmts = list(sqlparse.parsestream(stream))
assert isinstance(stmts[0].tokens[0].value, text_type)
def test_encoding_parsestream(self):
stream = StringIO("SELECT 1; SELECT 2;")
stmts = list(sqlparse.parsestream(stream))
self.assertEqual(type(stmts[0].tokens[0].value), text_type)
# -*- coding: utf-8 -*-
#
# Copyright © 2014 René Samselnig
#
# This file is part of Database Navigator.
#
# Database Navigator is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Database Navigator is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Database Navigator. If not, see <http://www.gnu.org/licenses/>.
#
import sys
import os
import sqlparse
#newin = os.fdopen(sys.stdin.fileno(), 'r', 1)
s = sqlparse.parsestream(sys.stdin)
for i in s:
print i
def test_encoding_parsestream(self):
stream = StringIO("SELECT 1; SELECT 2;")
stmts = list(sqlparse.parsestream(stream))
self.assertEqual(type(stmts[0].tokens[0].value), text_type)
"""
Downloads the following:
- Korean Wikipedia texts
- Korean
"""
from sqlparse import parsestream
from sqlparse.sql import Parenthesis
for statement in parsestream(open('data/test.sql')):
texts = [str(token.tokens[1].tokens[-1]).decode('string_escape') for token in statement.tokens if isinstance(token, Parenthesis)]
print texts
texts = [text for text in texts if text[0] != '#']
if texts:
print "\n===\n".join(texts)
"""
Downloads the following:
- Korean Wikipedia texts
- Korean
"""
from sqlparse import parsestream
from sqlparse.sql import Parenthesis
for statement in parsestream(open('data/test.sql')):
texts = [
str(token.tokens[1].tokens[-1]).decode('string_escape')
for token in statement.tokens if isinstance(token, Parenthesis)
]
print texts
texts = [text for text in texts if text[0] != '#']
if texts:
print "\n===\n".join(texts)
def test_encoding_parsestream(self):
from cStringIO import StringIO
stream = StringIO("SELECT 1; SELECT 2;")
stmts = list(sqlparse.parsestream(stream))
self.assertEqual(type(stmts[0].tokens[0].value), unicode)
def test_split_stream():
stream = StringIO("SELECT 1; SELECT 2;")
stmts = sqlparse.parsestream(stream)
assert isinstance(stmts, types.GeneratorType)
assert len(list(stmts)) == 2
def test_encoding_parsestream(self):
from cStringIO import StringIO
stream = StringIO("SELECT 1; SELECT 2;")
stmts = list(sqlparse.parsestream(stream))
self.assertEqual(type(stmts[0].tokens[0].value), unicode)
def source_stream(self) -> sqlparse.sql.Statement:
"""Parses source sql into individual statements."""
for s in sqlparse.parsestream(stream=self.source):
if self.sn.cfg.script.is_valid_sql(s=s):
yield s
