def _handle_DQL(self, sql, params):
with self.connection.database.snapshot() as snapshot:
# Reference
# https://googleapis.dev/python/spanner/latest/session-api.html#google.cloud.spanner_v1.session.Session.execute_sql
sql, params = parse_utils.sql_pyformat_args_to_spanner(sql, params)
res = snapshot.execute_sql(
sql, params=params, param_types=get_param_types(params)
)
if type(res) == int:
self._row_count = res
self._itr = None
else:
# Immediately using:
# iter(response)
# here, because this Spanner API doesn't provide
# easy mechanisms to detect when only a single item
# is returned or many, yet mixing results that
# are for .fetchone() with those that would result in
# many items returns a RuntimeError if .fetchone() is
# invoked and vice versa.
self._result_set = res
# Read the first element so that the StreamedResultSet can
# return the metadata after a DQL statement. See issue #155.
while True:
try:
self._itr = PeekIterator(self._result_set)
break
except Aborted:
self.connection.retry_transaction()
# Unfortunately, Spanner doesn't seem to send back
# information about the number of rows available.
self._row_count = _UNSET_COUNT
def _execute_insert_heterogenous(transaction, sql_params_list):
for sql, params in sql_params_list:
sql, params = sql_pyformat_args_to_spanner(sql, params)
param_types = get_param_types(params)
res = transaction.execute_sql(sql,
params=params,
param_types=param_types)
# TODO: File a bug with Cloud Spanner and the Python client maintainers
# about a lost commit when res isn't read from.
_ = list(res)
def test_sql_pyformat_args_to_spanner(self):
from google.cloud.spanner_dbapi.parse_utils import sql_pyformat_args_to_spanner
cases = [
(
(
"SELECT * from t WHERE f1=%s, f2 = %s, f3=%s",
(10, "abc", "y**$22l3f"),
),
(
"SELECT * from t WHERE f1=@a0, f2 = @a1, f3=@a2",
{"a0": 10, "a1": "abc", "a2": "y**$22l3f"},
),
),
(
(
"INSERT INTO t (f1, f2, f2) VALUES (%s, %s, %s)",
("app", "name", "applied"),
),
(
"INSERT INTO t (f1, f2, f2) VALUES (@a0, @a1, @a2)",
{"a0": "app", "a1": "name", "a2": "applied"},
),
),
(
(
"INSERT INTO t (f1, f2, f2) VALUES (%(f1)s, %(f2)s, %(f3)s)",
{"f1": "app", "f2": "name", "f3": "applied"},
),
(
"INSERT INTO t (f1, f2, f2) VALUES (@a0, @a1, @a2)",
{"a0": "app", "a1": "name", "a2": "applied"},
),
),
(
# Intentionally using a dict with more keys than will be resolved.
("SELECT * from t WHERE f1=%(f1)s", {"f1": "app", "f2": "name"}),
("SELECT * from t WHERE f1=@a0", {"a0": "app"}),
),
(
# No args to replace, we MUST return the original params dict
# since it might be useful to pass to the next user.
("SELECT * from t WHERE id=10", {"f1": "app", "f2": "name"}),
("SELECT * from t WHERE id=10", {"f1": "app", "f2": "name"}),
),
]
for ((sql_in, params), sql_want) in cases:
with self.subTest(sql=sql_in):
got_sql, got_named_args = sql_pyformat_args_to_spanner(sql_in, params)
want_sql, want_named_args = sql_want
self.assertEqual(got_sql, want_sql, "SQL does not match")
self.assertEqual(
got_named_args, want_named_args, "Named args do not match"
)
def _do_execute_update(self, transaction, sql, params, param_types=None):
sql = parse_utils.ensure_where_clause(sql)
sql, params = parse_utils.sql_pyformat_args_to_spanner(sql, params)
result = transaction.execute_update(
sql, params=params, param_types=get_param_types(params))
self._itr = None
if type(result) == int:
self._row_count = result
return result
def test_sql_pyformat_args_to_spanner_invalid(self):
cases = [(
"SELECT * from t WHERE f1=%s, f2 = %s, f3=%s, extra=%s",
(10, "abc", "y**$22l3f"),
)]
for sql, params in cases:
with self.subTest(sql=sql):
self.assertRaisesRegex(
Error,
"pyformat_args mismatch",
lambda: sql_pyformat_args_to_spanner(sql, params),
)
def execute(self, sql, args=None):
"""Prepares and executes a Spanner database operation.
:type sql: str
:param sql: A SQL query statement.
:type args: list
:param args: Additional parameters to supplement the SQL query.
"""
if not self.connection:
raise ProgrammingError("Cursor is not connected to the database")
self._raise_if_closed()
self._result_set = None
# Classify whether this is a read-only SQL statement.
try:
classification = parse_utils.classify_stmt(sql)
if classification == parse_utils.STMT_DDL:
self.connection._ddl_statements.append(sql)
return
# For every other operation, we've got to ensure that
# any prior DDL statements were run.
# self._run_prior_DDL_statements()
self.connection.run_prior_DDL_statements()
if not self.connection.autocommit:
transaction = self.connection.transaction_checkout()
sql, params = parse_utils.sql_pyformat_args_to_spanner(
sql, args)
self._result_set = transaction.execute_sql(
sql, params, param_types=get_param_types(params))
self._itr = PeekIterator(self._result_set)
return
if classification == parse_utils.STMT_NON_UPDATING:
self._handle_DQL(sql, args or None)
elif classification == parse_utils.STMT_INSERT:
_helpers.handle_insert(self.connection, sql, args or None)
else:
self.connection.database.run_in_transaction(
self._do_execute_update, sql, args or None)
except (AlreadyExists, FailedPrecondition) as e:
raise IntegrityError(e.details if hasattr(e, "details") else e)
except InvalidArgument as e:
raise ProgrammingError(e.details if hasattr(e, "details") else e)
except InternalServerError as e:
raise OperationalError(e.details if hasattr(e, "details") else e)
def _handle_DQL(self, sql, params):
sql, params = parse_utils.sql_pyformat_args_to_spanner(sql, params)
if self.connection.read_only and not self.connection.autocommit:
# initiate or use the existing multi-use snapshot
self._handle_DQL_with_snapshot(
self.connection.snapshot_checkout(), sql, params
)
else:
# execute with single-use snapshot
with self.connection.database.snapshot(
**self.connection.staleness
) as snapshot:
self._handle_DQL_with_snapshot(snapshot, sql, params)
def test_sql_pyformat_args_to_spanner_invalid(self):
from google.cloud.spanner_dbapi import exceptions
from google.cloud.spanner_dbapi.parse_utils import sql_pyformat_args_to_spanner
cases = [(
"SELECT * from t WHERE f1=%s, f2 = %s, f3=%s, extra=%s",
(10, "abc", "y**$22l3f"),
)]
for sql, params in cases:
with self.subTest(sql=sql):
self.assertRaisesRegex(
exceptions.Error,
"pyformat_args mismatch",
lambda: sql_pyformat_args_to_spanner(sql, params),
)
def _execute_insert_heterogenous(transaction, sql_params_list):
for sql, params in sql_params_list:
sql, params = sql_pyformat_args_to_spanner(sql, params)
transaction.execute_update(sql, params, get_param_types(params))
def executemany(self, operation, seq_of_params):
"""Execute the given SQL with every parameters set
from the given sequence of parameters.
:type operation: str
:param operation: SQL code to execute.
:type seq_of_params: list
:param seq_of_params: Sequence of additional parameters to run
the query with.
"""
self._raise_if_closed()
classification = parse_utils.classify_stmt(operation)
if classification == parse_utils.STMT_DDL:
raise ProgrammingError(
"Executing DDL statements with executemany() method is not allowed."
)
many_result_set = StreamedManyResultSets()
if classification in (parse_utils.STMT_INSERT, parse_utils.STMT_UPDATING):
statements = []
for params in seq_of_params:
sql, params = parse_utils.sql_pyformat_args_to_spanner(
operation, params
)
statements.append((sql, params, get_param_types(params)))
if self.connection.autocommit:
self.connection.database.run_in_transaction(
self._do_batch_update, statements, many_result_set
)
else:
retried = False
while True:
try:
transaction = self.connection.transaction_checkout()
res_checksum = ResultsChecksum()
if not retried:
self.connection._statements.append(
(statements, res_checksum)
)
status, res = transaction.batch_update(statements)
many_result_set.add_iter(res)
res_checksum.consume_result(res)
res_checksum.consume_result(status.code)
if status.code == ABORTED:
self.connection._transaction = None
raise Aborted(status.details)
elif status.code != OK:
raise OperationalError(status.details)
break
except Aborted:
self.connection.retry_transaction()
retried = True
else:
for params in seq_of_params:
self.execute(operation, params)
many_result_set.add_iter(self._itr)
self._result_set = many_result_set
self._itr = many_result_set
def execute(self, sql, args=None):
"""Prepares and executes a Spanner database operation.
:type sql: str
:param sql: A SQL query statement.
:type args: list
:param args: Additional parameters to supplement the SQL query.
"""
if not self.connection:
raise ProgrammingError("Cursor is not connected to the database")
self._raise_if_closed()
self._result_set = None
# Classify whether this is a read-only SQL statement.
try:
classification = parse_utils.classify_stmt(sql)
if classification == parse_utils.STMT_DDL:
ddl_statements = []
for ddl in sqlparse.split(sql):
if ddl:
if ddl[-1] == ";":
ddl = ddl[:-1]
if parse_utils.classify_stmt(ddl) != parse_utils.STMT_DDL:
raise ValueError("Only DDL statements may be batched.")
ddl_statements.append(ddl)
# Only queue DDL statements if they are all correctly classified.
self.connection._ddl_statements.extend(ddl_statements)
if self.connection.autocommit:
self.connection.run_prior_DDL_statements()
return
# For every other operation, we've got to ensure that
# any prior DDL statements were run.
# self._run_prior_DDL_statements()
self.connection.run_prior_DDL_statements()
if not self.connection.autocommit:
if classification == parse_utils.STMT_UPDATING:
sql = parse_utils.ensure_where_clause(sql)
if classification != parse_utils.STMT_INSERT:
sql, args = sql_pyformat_args_to_spanner(sql, args or None)
statement = Statement(
sql,
args,
get_param_types(args or None)
if classification != parse_utils.STMT_INSERT
else {},
ResultsChecksum(),
classification == parse_utils.STMT_INSERT,
)
(self._result_set, self._checksum,) = self.connection.run_statement(
statement
)
while True:
try:
self._itr = PeekIterator(self._result_set)
break
except Aborted:
self.connection.retry_transaction()
return
if classification == parse_utils.STMT_NON_UPDATING:
self._handle_DQL(sql, args or None)
elif classification == parse_utils.STMT_INSERT:
_helpers.handle_insert(self.connection, sql, args or None)
else:
self.connection.database.run_in_transaction(
self._do_execute_update, sql, args or None
)
except (AlreadyExists, FailedPrecondition) as e:
raise IntegrityError(e.details if hasattr(e, "details") else e)
except InvalidArgument as e:
raise ProgrammingError(e.details if hasattr(e, "details") else e)
except InternalServerError as e:
raise OperationalError(e.details if hasattr(e, "details") else e)
def execute(self, sql, args=None):
"""Prepares and executes a Spanner database operation.
:type sql: str
:param sql: A SQL query statement.
:type args: list
:param args: Additional parameters to supplement the SQL query.
"""
self._result_set = None
try:
if self.connection.read_only:
self._handle_DQL(sql, args or None)
return
class_ = parse_utils.classify_stmt(sql)
if class_ == parse_utils.STMT_DDL:
self._batch_DDLs(sql)
if self.connection.autocommit:
self.connection.run_prior_DDL_statements()
return
# For every other operation, we've got to ensure that
# any prior DDL statements were run.
# self._run_prior_DDL_statements()
self.connection.run_prior_DDL_statements()
if class_ == parse_utils.STMT_UPDATING:
sql = parse_utils.ensure_where_clause(sql)
if class_ != parse_utils.STMT_INSERT:
sql, args = sql_pyformat_args_to_spanner(sql, args or None)
if not self.connection.autocommit:
statement = Statement(
sql,
args,
get_param_types(args or None)
if class_ != parse_utils.STMT_INSERT
else {},
ResultsChecksum(),
class_ == parse_utils.STMT_INSERT,
)
(
self._result_set,
self._checksum,
) = self.connection.run_statement(statement)
while True:
try:
self._itr = PeekIterator(self._result_set)
break
except Aborted:
self.connection.retry_transaction()
return
if class_ == parse_utils.STMT_NON_UPDATING:
self._handle_DQL(sql, args or None)
elif class_ == parse_utils.STMT_INSERT:
_helpers.handle_insert(self.connection, sql, args or None)
else:
self.connection.database.run_in_transaction(
self._do_execute_update, sql, args or None
)
except (AlreadyExists, FailedPrecondition, OutOfRange) as e:
raise IntegrityError(getattr(e, "details", e))
except InvalidArgument as e:
raise ProgrammingError(getattr(e, "details", e))
except InternalServerError as e:
raise OperationalError(getattr(e, "details", e))
