def select_by_ids(class_,
cursor,
where_column_value_pairs,
columns_list=None,
order_by=None,
group_by=None,
limit=None,
**kwargs):
"""This method is used to perform in-clause queries.
Such queries can cause vtgate to scatter over multiple shards.
This uses execute_entity_ids method of vtgate cursor and the entity
column and the associated entity_keyspace_id_map is computed based
on the routing used - sharding_key or entity_id_map.
"""
if class_.columns_list is None:
raise dbexceptions.ProgrammingError(
"DB class should define columns_list")
if columns_list is None:
columns_list = class_.columns_list
query, bind_vars = sql_builder.select_by_columns_query(
columns_list,
class_.table_name,
where_column_value_pairs,
order_by=order_by,
group_by=group_by,
limit=limit,
**kwargs)
entity_col_name = None
entity_id_keyspace_id_map = {}
if cursor.routing.sharding_key is not None:
# If the in-clause is based on sharding key
entity_col_name = class_.sharding_key_column_name
if db_object._is_iterable_container(cursor.routing.sharding_key):
for sk in list(cursor.routing.sharding_key):
entity_id_keyspace_id_map[sk] = pack_keyspace_id(
class_.sharding_key_to_keyspace_id(sk))
else:
sk = cursor.routing.sharding_key
entity_id_keyspace_id_map[sk] = pack_keyspace_id(
class_.sharding_key_to_keyspace_id(sk))
elif cursor.routing.entity_id_sharding_key_map is not None:
# If the in-clause is based on entity column
entity_col_name = cursor.routing.entity_column_name
for en_id, sk in cursor.routing.entity_id_sharding_key_map.iteritems(
):
entity_id_keyspace_id_map[en_id] = pack_keyspace_id(
class_.sharding_key_to_keyspace_id(sk))
else:
dbexceptions.ProgrammingError("Invalid routing method used.")
# cursor.routing.entity_column_name is set while creating shard routing.
rowcount = cursor.execute_entity_ids(query, bind_vars,
entity_id_keyspace_id_map,
entity_col_name)
rows = cursor.fetchall()
return [sql_builder.DBRow(columns_list, row) for row in rows]
def _db_wrapper(*pargs, **kwargs):
table_class = pargs[0]
write_method = kwargs.get('write_method', False)
if not issubclass(table_class, DBObjectBase):
raise dbexceptions.ProgrammingError(
"table class '%s' is not inherited from DBObjectBase" %
table_class)
# Create the cursor using cursor_method
cursor_method = pargs[1]
cursor = cursor_method(table_class)
# DML verification.
if write_method:
if not cursor.is_writable():
raise dbexceptions.ProgrammingError(
'Executing dmls on a non-writable cursor is not allowed.')
if table_class.is_mysql_view:
raise dbexceptions.ProgrammingError('writes disabled on view',
table_class)
if pargs[2:]:
return method(table_class, cursor, *pargs[2:], **kwargs)
else:
return method(table_class, cursor, **kwargs)
def delete_by_columns(class_,
cursor,
where_column_value_pairs,
limit=None):
sharding_key = cursor.routing.sharding_key
if sharding_key is None:
raise dbexceptions.ProgrammingError("sharding_key cannot be empty")
if not where_column_value_pairs:
raise dbexceptions.ProgrammingError(
"deleting the whole table is not allowed")
query, bind_vars = sql_builder.delete_by_columns_query(
class_.table_name, where_column_value_pairs, limit=limit)
cursor.execute(query, bind_vars)
if cursor.rowcount == 0:
raise dbexceptions.DatabaseError("DB Row not found")
rowcount = cursor.rowcount
#delete the lookup map.
lookup_cursor_method = functools.partial(
db_object.create_cursor_from_old_cursor, cursor)
class_.delete_sharding_key_entity_id_lookup(lookup_cursor_method,
sharding_key)
return rowcount
def execute_batch_read(cursor, query_list, bind_vars_list):
"""Method for executing select queries in batch.
Args:
cursor: original cursor - that is converted to read-only BatchVTGateCursor.
query_list: query_list.
bind_vars_list: bind variables list.
Returns:
Result of the form [[q1row1, q1row2,...], [q2row1, ...],..]
Raises:
dbexceptions.ProgrammingError when dmls are issued to read batch cursor.
"""
if not isinstance(cursor, vtgate_cursor.VTGateCursor):
raise dbexceptions.ProgrammingError(
"cursor is not of the type VTGateCursor.")
batch_cursor = create_batch_cursor_from_cursor(cursor)
for q, bv in zip(query_list, bind_vars_list):
if is_dml(q):
raise dbexceptions.ProgrammingError("Dml %s for read batch cursor." % q)
batch_cursor.execute(q, bv)
batch_cursor.flush()
rowsets = batch_cursor.rowsets
result = []
# rowset is of the type [(results, rowcount, lastrowid, fields),..]
for rowset in rowsets:
rowset_results = rowset[0]
fields = [f[0] for f in rowset[3]]
rows = []
for row in rowset_results:
rows.append(sql_builder.DBRow(fields, row))
result.append(rows)
return result
def create_where_clause_for_keyrange(
keyrange,
keyspace_col_name='keyspace_id',
keyspace_col_type=keyrange_constants.KIT_UINT64):
if isinstance(keyrange, str):
# If the keyrange is for unsharded db, there is no
# where clause to add to or bind_vars to add to.
if keyrange == keyrange_constants.NON_PARTIAL_KEYRANGE:
return "", {}
keyrange = keyrange.split('-')
if not isinstance(keyrange, tuple) and not isinstance(
keyrange, list) or len(keyrange) != 2:
raise dbexceptions.ProgrammingError(
"keyrange must be a list or tuple or a '-' separated str %s" %
keyrange)
if keyspace_col_type == keyrange_constants.KIT_UINT64:
return _create_where_clause_for_int_keyspace(keyrange,
keyspace_col_name)
elif keyspace_col_type == keyrange_constants.KIT_BYTES:
return _create_where_clause_for_str_keyspace(keyrange,
keyspace_col_name)
else:
raise dbexceptions.ProgrammingError(
"Illegal type for keyspace_col_type %d" % keyspace_col_type)
def execute_batch_write(cursor, query_list, bind_vars_list):
"""Method for executing dml queries in batch.
Args:
cursor: original cursor - that is converted to read-only BatchVTGateCursor.
query_list: query_list.
bind_vars_list: bind variables list.
Returns:
Result of the form [{'rowcount':rowcount, 'lastrowid':lastrowid}, ...]
since for dmls those two values are valuable.
Raises:
dbexceptions.ProgrammingError when non-dmls are issued to writable batch cursor.
"""
if not isinstance(cursor, vtgate_cursor.VTGateCursor):
raise dbexceptions.ProgrammingError(
"cursor is not of the type VTGateCursor.")
batch_cursor = create_batch_cursor_from_cursor(cursor, writable=True)
if batch_cursor.is_writable() and len(batch_cursor.keyspace_ids) != 1:
raise dbexceptions.ProgrammingError(
"writable batch execute can also execute on one keyspace_id.")
for q, bv in zip(query_list, bind_vars_list):
if not is_dml(q):
raise dbexceptions.ProgrammingError("query %s is not a dml" % q)
batch_cursor.execute(q, bv)
batch_cursor.flush()
rowsets = batch_cursor.rowsets
result = []
# rowset is of the type [(results, rowcount, lastrowid, fields),..]
for rowset in rowsets:
result.append({'rowcount': rowset[1], 'lastrowid': rowset[2]})
return result
def execute(self, sql, bind_variables, **kargs):
self.rowcount = 0
self.results = None
self.description = None
self.lastrowid = None
sql_check = sql.strip().lower()
if sql_check == 'begin':
self.begin()
return
elif sql_check == 'commit':
self.commit()
return
elif sql_check == 'rollback':
self.rollback()
return
write_query = bool(write_sql_pattern.match(sql))
# NOTE: This check may also be done at high-layers but adding it here for completion.
if write_query:
if not self.is_writable():
raise dbexceptions.DatabaseError(
'DML on a non-writable cursor', sql)
# FIXME(shrutip): these checks should be on vtgate server, so
# dependency on topology can be removed.
if topology.is_sharded_keyspace(self.keyspace, self.tablet_type):
if self.keyspace_ids is None or len(self.keyspace_ids) != 1:
raise dbexceptions.ProgrammingError(
'DML on zero or multiple keyspace ids is not allowed: %r'
% self.keyspace_ids)
else:
if not self.keyranges or str(
self.keyranges[0]
) != keyrange_constants.NON_PARTIAL_KEYRANGE:
raise dbexceptions.ProgrammingError(
'Keyrange not correct for non-sharded keyspace: %r' %
self.keyranges)
self.results, self.rowcount, self.lastrowid, self.description = self._conn._execute(
sql,
bind_variables,
self.keyspace,
self.tablet_type,
keyspace_ids=self.keyspace_ids,
keyranges=self.keyranges)
self.index = 0
return self.rowcount
def select_by_columns(class_,
cursor,
where_column_value_pairs,
columns_list=None,
order_by=None,
group_by=None,
limit=None,
**kwargs):
if class_.columns_list is None:
raise dbexceptions.ProgrammingError(
"DB class should define columns_list")
if columns_list is None:
columns_list = class_.columns_list
query, bind_vars = sql_builder.select_by_columns_query(
columns_list,
class_.table_name,
where_column_value_pairs,
order_by=order_by,
group_by=group_by,
limit=limit,
**kwargs)
rowcount = cursor.execute(query, bind_vars)
rows = cursor.fetchall()
return [sql_builder.DBRow(columns_list, row) for row in rows]
def convert_exception(exc, *args):
new_args = exc.args + args
if isinstance(exc, gorpc.TimeoutError):
return dbexceptions.TimeoutError(new_args)
elif isinstance(exc, gorpc.AppError):
msg = str(exc[0]).lower()
if msg.startswith('retry'):
return dbexceptions.RetryError(new_args)
if msg.startswith('fatal'):
return dbexceptions.FatalError(new_args)
if msg.startswith('tx_pool_full'):
return dbexceptions.TxPoolFull(new_args)
match = _errno_pattern.search(msg)
if match:
mysql_errno = int(match.group(1))
if mysql_errno == 1062:
return dbexceptions.IntegrityError(new_args)
# TODO(sougou/liguo): remove this case once servers are deployed
elif mysql_errno == 1290 and 'read-only' in msg:
return dbexceptions.RetryError(new_args)
return dbexceptions.DatabaseError(new_args)
elif isinstance(exc, gorpc.ProgrammingError):
return dbexceptions.ProgrammingError(new_args)
elif isinstance(exc, gorpc.GoRpcError):
return dbexceptions.FatalError(new_args)
return exc
def convert_exception(exc, *args, **kwargs):
"""This parses the protocol exceptions to the api interface exceptions.
This also logs the exception and increments the appropriate error counters.
Args:
exc: raw protocol exception.
args: additional args from the raising site.
kwargs: additional keyword args from the raising site.
Returns:
Api interface exceptions - dbexceptions with new args.
"""
new_args = exc.args + args
if kwargs:
new_args += tuple(kwargs.itervalues())
new_exc = exc
if isinstance(exc, gorpc.TimeoutError):
new_exc = dbexceptions.TimeoutError(new_args)
elif isinstance(exc, gorpc.AppError):
new_exc = handle_app_error(new_args)
elif isinstance(exc, gorpc.ProgrammingError):
new_exc = dbexceptions.ProgrammingError(new_args)
elif isinstance(exc, gorpc.GoRpcError):
new_exc = dbexceptions.FatalError(new_args)
keyspace = kwargs.get("keyspace", None)
tablet_type = kwargs.get("tablet_type", None)
vtgate_utils.log_exception(new_exc,
keyspace=keyspace,
tablet_type=tablet_type)
return new_exc
def _stream_execute(self, sql, bind_variables, keyspace, tablet_type, keyspace_ids=None, keyranges=None):
exec_method = None
req = None
if keyspace_ids is not None:
req = _create_req_with_keyspace_ids(sql, bind_variables, keyspace, tablet_type, keyspace_ids)
exec_method = 'VTGate.StreamExecuteKeyspaceIds'
elif keyranges is not None:
req = _create_req_with_keyranges(sql, bind_variables, keyspace, tablet_type, keyranges)
exec_method = 'VTGate.StreamExecuteKeyRanges'
else:
raise dbexceptions.ProgrammingError('_stream_execute called without specifying keyspace_ids or keyranges')
self._add_session(req)
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call(exec_method, req)
first_response = self.client.stream_next()
reply = first_response.reply['Result']
for field in reply['Fields']:
self._stream_fields.append((field['Name'], field['Type']))
self._stream_conversions.append(field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return None, 0, 0, self._stream_fields
def _convert_exception(self, exc, *args, **kwargs):
"""This parses the protocol exceptions to the api interface exceptions.
This also logs the exception and increments the appropriate error counters.
Args:
exc: raw protocol exception.
*args: additional args from the raising site.
**kwargs: additional keyword args from the raising site.
Returns:
Api interface exceptions - dbexceptions with new args.
"""
kwargs_as_str = vtgate_utils.convert_exception_kwargs(kwargs)
exc.args += args
if kwargs_as_str:
exc.args += kwargs_as_str,
new_args = (type(exc).__name__,) + exc.args
if isinstance(exc, gorpc.TimeoutError):
new_exc = dbexceptions.TimeoutError(new_args)
elif isinstance(exc, vtgate_utils.VitessError):
new_exc = exc.convert_to_dbexception(new_args)
elif isinstance(exc, gorpc.ProgrammingError):
new_exc = dbexceptions.ProgrammingError(new_args)
elif isinstance(exc, gorpc.GoRpcError):
new_exc = dbexceptions.FatalError(new_args)
else:
new_exc = exc
vtgate_utils.log_exception(
new_exc,
keyspace=kwargs.get('keyspace'), tablet_type=kwargs.get('tablet_type'))
return new_exc
def create_shard_routing(class_, is_dml, **kargs):
routing = ShardRouting(class_.keyspace)
keyrange = kargs.get("keyrange", None)
if keyrange is not None:
if is_dml:
dbexceptions.InternalError(
"Writes require unique sharding_key and not keyrange.")
routing.keyrange = keyrange
return routing
routing.sharding_key = kargs.get('sharding_key', None)
if routing.sharding_key is None:
try:
entity_id_column = kargs['entity_id_column']
entity_id = kargs['entity_id']
# this may involve a lookup of the index from db.
# consider caching it at the app layer for performance.
entity_id_sharding_key_map = class_.map_entity_id_sharding_key(
entity_id_column, entity_id)
routing.entity_id_sharding_key_map = entity_id_sharding_key_map
routing = entity_id_sharding_key_map.values()
except KeyError, e:
raise dbexceptions.ProgrammingError(
"For sharded table, sharding_key and entity_id cannot both be empty.")
def _convert_exception(self, exc, *args, **kwargs):
"""This parses the protocol exceptions to the api interface exceptions.
This also logs the exception and increments the appropriate error counters.
Args:
exc: raw protocol exception.
*args: additional args from the raising site.
**kwargs: additional keyword args from the raising site.
Returns:
Api interface exceptions - dbexceptions with new args.
"""
new_args = exc.args + (str(self), ) + args
if kwargs:
new_args += tuple(sorted(kwargs.itervalues()))
new_exc = exc
if isinstance(exc, gorpc.TimeoutError):
new_exc = dbexceptions.TimeoutError(new_args)
elif isinstance(exc, vtgate_utils.VitessError):
new_exc = exc.convert_to_dbexception(new_args)
elif isinstance(exc, gorpc.ProgrammingError):
new_exc = dbexceptions.ProgrammingError(new_args)
elif isinstance(exc, gorpc.GoRpcError):
new_exc = dbexceptions.FatalError(new_args)
keyspace_name = kwargs.get('keyspace', None)
tablet_type = kwargs.get('tablet_type', None)
vtgate_utils.log_exception(new_exc,
keyspace=keyspace_name,
tablet_type=tablet_type)
return new_exc
def insert_primary(class_, cursor, **bind_vars):
if class_.columns_list is None:
raise dbexceptions.ProgrammingError('DB class should define columns_list')
query, bind_vars = class_.create_insert_query(**bind_vars)
cursor.execute(query, bind_vars)
return cursor.lastrowid
def convert_to_dbexception(self, args):
"""Converts from a VitessError to the appropriate dbexceptions class.
Args:
args: argument tuple to use to create the new exception.
Returns:
An exception from dbexceptions.
"""
# FIXME(alainjobart): this is extremely confusing: self.message is only
# used for integrity errors, and nothing else. The other cases
# have to provide the message in the args.
if self.code == vtrpc_pb2.UNAVAILABLE:
if throttler_err_re.search(self.message):
return dbexceptions.ThrottledError(args)
return dbexceptions.TransientError(args)
if self.code == vtrpc_pb2.FAILED_PRECONDITION:
return dbexceptions.QueryNotServed(args)
if self.code == vtrpc_pb2.ALREADY_EXISTS:
# Prune the error message to truncate after the mysql errno, since
# the error message may contain the query string with bind variables.
msg = self.message.lower()
parts = self._errno_pattern.split(msg)
pruned_msg = msg[:msg.find(parts[2])]
new_args = (pruned_msg, ) + tuple(args[1:])
return dbexceptions.IntegrityError(new_args)
if self.code == vtrpc_pb2.INVALID_ARGUMENT:
return dbexceptions.ProgrammingError(args)
return dbexceptions.DatabaseError(args)
def get_max(class_, cursor):
if class_.id_column_name is None:
raise dbexceptions.ProgrammingError('id_column_name not set.')
query, bind_vars = sql_builder.build_aggregate_query(
class_.table_name, class_.id_column_name, is_asc=False)
cursor.execute(query, bind_vars)
return cursor.fetch_aggregate_function(max)
def create_streaming_task_map(num_tasks, global_shard_count):
# global_shard_count is a configurable value controlled for resharding.
if num_tasks < global_shard_count:
raise dbexceptions.ProgrammingError("Tasks %d cannot be less than number of shards %d" % (num_tasks, global_shard_count))
stm = StreamingTaskMap(num_tasks)
stm.compute_kr_list()
return stm
def fetchone(self):
if self.description is None:
raise dbexceptions.ProgrammingError('Fetch called before execute.')
self.index += 1
try:
return self.generator.next()
except StopIteration:
return None
def fetchone(self):
if self.results is None:
raise dbexceptions.ProgrammingError('fetch called before execute')
if self.index >= len(self.results):
return None
self.index += 1
return self.results[self.index - 1]
def create_shard_routing(class_, *pargs, **kargs):
"""This creates the ShardRouting object based on the kargs.
This prunes the routing kargs so as not to interfere with the
actual database method.
Args:
*pargs: Positional arguments
**kargs: Routing key-value params. These are used to determine routing.
There are two mutually exclusive mechanisms to indicate routing.
1. entity_id_map {"entity_id_column": entity_id_value} where entity_id_column
could be the sharding key or a lookup based entity column of this table. This
helps determine the keyspace_ids for the cursor.
2. keyrange - This helps determine the keyrange for the cursor.
Returns:
ShardRouting object and modified kargs
"""
lookup_cursor_method = pargs[0]
routing = db_object.ShardRouting(class_.keyspace)
entity_id_map = None
entity_id_map = kargs.get("entity_id_map", None)
if entity_id_map is None:
kr = None
key_range = kargs.get("keyrange", None)
if isinstance(key_range, keyrange.KeyRange):
kr = key_range
else:
kr = keyrange.KeyRange(key_range)
if kr is not None:
routing.keyrange = kr
# Both entity_id_map and keyrange have been evaluated. Return.
return routing
# entity_id_map is not None
if len(entity_id_map) != 1:
dbexceptions.ProgrammingError("Invalid entity_id_map '%s'" %
entity_id_map)
entity_id_col = entity_id_map.keys()[0]
entity_id = entity_id_map[entity_id_col]
#TODO: the current api means that if a table doesn't have the sharding key column name
# then it cannot pass in sharding key for routing purposes. Will this cause
# extra load on lookup db/cache ? This is cleaner from a design perspective.
if entity_id_col == class_.sharding_key_column_name:
# Routing using sharding key.
routing.sharding_key = entity_id
if not class_.is_sharding_key_valid(routing.sharding_key):
raise dbexceptions.InternalError("Invalid sharding_key %s" %
routing.sharding_key)
else:
# Routing using lookup based entity.
routing.entity_column_name = entity_id_col
routing.entity_id_sharding_key_map = class_.lookup_sharding_key_from_entity_id(
lookup_cursor_method, entity_id_col, entity_id)
return routing
def insert(class_, cursor_method, **bind_vars):
""" This method creates the lookup relationship as well as the insert
in the primary table. The creation of the lookup entry also creates the
primary key for the row in the primary table.
The lookup relationship is determined by class_.column_lookup_name_map and the bind
variables passed in. There are two types of entities -
1. Table for which the entity that is also the primary sharding key for this keyspace.
2. Entity table that creates a new entity and needs to create a lookup between
that entity and sharding key.
"""
if class_.sharding_key_column_name is None:
raise dbexceptions.ProgrammingError(
"sharding_key_column_name empty for DBObjectEntityRangeSharded"
)
# Used for insert into class_.table_name
new_inserted_key = None
# Used for routing the insert_primary
entity_id_map = {}
# Create the lookup entry first
if class_.sharding_key_column_name in bind_vars:
# Secondary entity creation
sharding_key = bind_vars[class_.sharding_key_column_name]
entity_col = class_.entity_id_lookup_map.keys()[0]
lookup_bind_vars = {class_.sharding_key_column_name, sharding_key}
entity_id = class_.get_insert_id_from_lookup(
cursor_method, entity_col, **lookup_bind_vars)
bind_vars[entity_col] = entity_id
new_inserted_key = entity_id
entity_id_map[entity_col] = entity_id
else:
# Primary sharding key creation
# FIXME: what if class_.entity_id_lookup_map was empty ?
# there would need to be some table on which there was an auto-inc
# to generate the primary sharding key.
entity_col = class_.entity_id_lookup_map.keys()[0]
entity_id = bind_vars[entity_col]
lookup_bind_vars = {entity_col: entity_id}
sharding_key = class_.get_insert_id_from_lookup(
cursor_method, entity_col, **lookup_bind_vars)
bind_vars[class_.sharding_key_column_name] = sharding_key
new_inserted_key = sharding_key
entity_id_map[class_.sharding_key_column_name] = sharding_key
# FIXME: is the not value check correct ?
if 'keyspace_id' not in bind_vars or not bind_vars['keyspace_id']:
keyspace_id = class_.sharding_key_to_keyspace_id(sharding_key)
bind_vars['keyspace_id'] = keyspace_id
# entity_id_map is used for routing and hence passed to cursor_method
#bind_vars['entity_id_map'] = entity_id_map
new_cursor = functools.partial(cursor_method,
entity_id_map=entity_id_map)
class_.insert_primary(new_cursor, **bind_vars)
return new_inserted_key
def get_max(class_, cursor):
if class_.id_column_name is None:
raise dbexceptions.ProgrammingError("id_column_name not set.")
query = sql_builder.build_aggregate_query(class_.table_name,
class_.id_column_name,
sort_func='max')
cursor.execute(query, EmptyBindVariables)
return cursor.fetch_aggregate_function(max)
def insert(class_, cursor, **bind_vars):
if class_.columns_list is None:
raise dbexceptions.ProgrammingError("DB class should define columns_list")
query, bind_vars = sql_builder.insert_query(class_.table_name,
class_.columns_list,
**bind_vars)
cursor.execute(query, bind_vars)
return cursor.lastrowid
def create_cursor(self, writable, table_class, **cursor_kargs):
if not self.in_db_operation:
raise dbexceptions.ProgrammingError(
"Cannot execute queries outside db operations context.")
cursor = table_class.create_vtgate_cursor(self.get_vtgate_connection(),
self.tablet_type, writable,
**cursor_kargs)
return cursor
def execute(self, sql, bind_variables, **kwargs):
"""Perform a query, return the number of rows affected."""
self._clear_list_state()
self._clear_batch_state()
if self._handle_transaction_sql(sql):
return
entity_keyspace_id_map = kwargs.pop('entity_keyspace_id_map', None)
entity_column_name = kwargs.pop('entity_column_name', None)
write_query = bool(write_sql_pattern.match(sql))
# NOTE: This check may also be done at higher layers but adding it
# here for completion.
if write_query:
if not self.is_writable():
raise dbexceptions.ProgrammingError(
'DML on a non-writable cursor', sql)
if entity_keyspace_id_map:
raise dbexceptions.ProgrammingError(
'entity_keyspace_id_map is not allowed for write queries')
# FIXME(alainjobart): the entity_keyspace_id_map should be in the
# cursor, same as keyspace_ids, shards, keyranges, to avoid this hack.
if entity_keyspace_id_map:
shards = None
keyspace_ids = None
keyranges = None
else:
shards = self.shards
keyspace_ids = self.keyspace_ids
keyranges = self.keyranges
self.results, self.rowcount, self.lastrowid, self.description = (
self.connection._execute( # pylint: disable=protected-access
sql,
bind_variables,
tablet_type=self.tablet_type,
keyspace_name=self.keyspace,
shards=shards,
keyspace_ids=keyspace_ids,
keyranges=keyranges,
entity_keyspace_id_map=entity_keyspace_id_map,
entity_column_name=entity_column_name,
not_in_transaction=not self.is_writable(),
effective_caller_id=self.effective_caller_id,
**kwargs))
return self.rowcount
def delete_by_columns(class_, cursor, where_column_value_pairs, limit=None):
if not where_column_value_pairs:
raise dbexceptions.ProgrammingError("deleting the whole table is not allowed")
query, bind_vars = class_.create_delete_query(where_column_value_pairs,
limit=limit)
cursor.execute(query, bind_vars)
if cursor.rowcount == 0:
raise dbexceptions.DatabaseError("DB Row not found")
return cursor.rowcount
def convert_exception(exc, *args):
new_args = exc.args + args
if isinstance(exc, gorpc.TimeoutError):
return dbexceptions.TimeoutError(new_args)
elif isinstance(exc, gorpc.AppError):
return handle_app_error(new_args)
elif isinstance(exc, gorpc.ProgrammingError):
return dbexceptions.ProgrammingError(new_args)
elif isinstance(exc, gorpc.GoRpcError):
return dbexceptions.FatalError(new_args)
return exc
def fetchmany(self, size=None):
if self.results is None:
raise dbexceptions.ProgrammingError('fetch called before execute')
if self.index >= len(self.results):
return []
if size is None:
size = self.arraysize
res = self.results[self.index:self.index + size]
self.index += size
return res
def _execute(self,
sql,
bind_variables,
keyspace,
tablet_type,
keyspace_ids=None,
keyranges=None):
exec_method = None
req = None
if keyspace_ids is not None:
req = _create_req_with_keyspace_ids(sql, bind_variables, keyspace,
tablet_type, keyspace_ids)
exec_method = 'VTGate.ExecuteKeyspaceIds'
elif keyranges is not None:
req = _create_req_with_keyranges(sql, bind_variables, keyspace,
tablet_type, keyranges)
exec_method = 'VTGate.ExecuteKeyRanges'
else:
raise dbexceptions.ProgrammingError(
'_execute called without specifying keyspace_ids or keyranges')
self._add_session(req)
fields = []
conversions = []
results = []
rowcount = 0
lastrowid = 0
try:
response = self.client.call(exec_method, req)
self._update_session(response)
reply = response.reply
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'], exec_method)
if 'Result' in reply:
res = reply['Result']
for field in res['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in res['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = res['RowsAffected']
lastrowid = res['InsertId']
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
