def _create_v2_request_with_keyspace_ids(sql, new_binds, keyspace_name, tablet_type, keyspace_ids, not_in_transaction):
"""Make a request dict from arguments.
Args:
sql: Str sql with format tokens.
new_binds: Dict of bind variables.
keyspace_name: Str keyspace name.
tablet_type: Str tablet_type.
keyspace_ids: Bytes list of keyspace IDs.
not_in_transaction: Bool True if a transaction should not be started
(generally used when sql is not a write).
Returns:
A (str: value) dict.
"""
# keyspace_ids are Keyspace Ids packed to byte[]
sql, new_binds = dbapi.prepare_query_bind_vars(sql, new_binds)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
"Sql": sql,
"BindVariables": new_binds,
"Keyspace": keyspace_name,
"TabletType": topodata_pb2.TabletType.Value(tablet_type.upper()),
"KeyspaceIds": keyspace_ids,
"NotInTransaction": not_in_transaction,
}
return req
def _execute_batch(self,
sql_list,
bind_variables_list,
keyspace,
tablet_type,
keyspace_ids,
not_in_transaction=False):
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
sql, bind_vars = dbapi.prepare_query_bind_vars(sql, bind_vars)
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query_list.append(query)
rowsets = []
try:
req = {
'Queries': query_list,
'Keyspace': keyspace,
'TabletType': tablet_type,
'KeyspaceIds': keyspace_ids,
'NotInTransaction': not_in_transaction,
}
self._add_session(req)
response = self.client.call('VTGate.ExecuteBatchKeyspaceIds', req)
self._update_session(response)
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'],
'VTGate.ExecuteBatchKeyspaceIds')
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables_list)
raise convert_exception(e,
str(self),
sql_list,
keyspace_ids,
keyspace=keyspace,
tablet_type=tablet_type)
except:
logging.exception('gorpc low-level error')
raise
return rowsets
def _create_req_with_keyranges(sql, new_binds, keyspace_name, tablet_type,
keyranges, not_in_transaction):
"""Make a request dict from arguments.
Args:
sql: Str sql with format tokens.
new_binds: Dict of bind variables.
keyspace_name: Str keyspace name.
tablet_type: Str tablet_type.
keyranges: A list of keyrange.KeyRange objects.
not_in_transaction: Bool True if a transaction should not be started
(generally used when sql is not a write).
Returns:
A (str: value) dict.
"""
sql, new_binds = dbapi.prepare_query_bind_vars(sql, new_binds)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': keyspace_name,
'TabletType': tablet_type,
'KeyRanges': keyranges,
'NotInTransaction': not_in_transaction,
}
return req
def _stream_execute(self, sql, bind_variables):
req = {
'Query': {
'Sql': sql,
'BindVariables': field_types.convert_bind_vars(bind_variables),
'SessionId': self.session_id,
'TransactionId': self.transaction_id
},
'ImmediateCallerID': {'Username': self.caller_id}
}
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call('SqlQuery.StreamExecute2', req)
first_response = self.client.stream_next()
reply = first_response.reply
if reply.get('Err'):
self.__drain_conn_after_streaming_app_error()
raise gorpc.AppError(reply['Err'].get(
'Message', 'Missing error message'))
for field in reply['Fields']:
self._stream_fields.append((field['Name'], field['Type']))
self._stream_conversions.append(
field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return None, 0, 0, self._stream_fields
def _execute(self, sql, bind_variables):
req = {
'QueryRequest': {
'Sql': sql,
'BindVariables': field_types.convert_bind_vars(bind_variables),
'SessionId': self.session_id,
'TransactionId': self.transaction_id
},
'ImmediateCallerID': {'Username': self.caller_id}
}
fields = []
conversions = []
results = []
try:
response = self.rpc_call_and_extract_error('SqlQuery.Execute2', req)
reply = response.reply
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except Exception:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def _execute(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
req = self._make_req()
req["Sql"] = sql
req["BindVariables"] = new_binds
fields = []
conversions = []
results = []
try:
response = self.client.call("VTGate.ExecuteShard", req)
reply = response.reply
for field in reply["Fields"]:
fields.append((field["Name"], field["Type"]))
conversions.append(field_types.conversions.get(field["Type"]))
for row in reply["Rows"]:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply["RowsAffected"]
lastrowid = reply["InsertId"]
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql, bind_variables)
except:
logging.exception("gorpc low-level error")
raise
return results, rowcount, lastrowid, fields
def _execute(self, sql, bind_variables):
req = {
'QueryRequest': {
'Sql': sql,
'BindVariables': field_types.convert_bind_vars(bind_variables),
'SessionId': self.session_id,
'TransactionId': self.transaction_id
},
'ImmediateCallerID': {'Username': self.caller_id}
}
fields = []
conversions = []
results = []
try:
response = self.rpc_call_and_extract_error('SqlQuery.Execute2', req)
reply = response.reply
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
except (gorpc.GoRpcError, TabletError) as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except Exception:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def _stream_execute(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': self.keyspace,
'TabletType': self.tablet_type,
'Shards': [self.shard],
}
self._add_session(req)
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call('VTGate.StreamExecuteShard', req)
first_response = self.client.stream_next()
reply = first_response.reply
for field in reply['Fields']:
self._stream_fields.append((field['Name'], field['Type']))
self._stream_conversions.append(field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql, bind_variables)
except:
logging.exception('gorpc low-level error')
raise
return None, 0, 0, self._stream_fields
def _execute(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
req = self._make_req()
req['Sql'] = sql
req['BindVariables'] = new_binds
fields = []
conversions = []
results = []
try:
response = self.client.call('Barnacle.Execute', req)
reply = response.reply
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql, bind_variables)
except:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def _stream_execute2(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
query = self._make_req()
query['Sql'] = sql
query['BindVariables'] = new_binds
req = {'Query': query}
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call('SqlQuery.StreamExecute2', req)
first_response = self.client.stream_next()
reply = first_response.reply
if reply.get('Err'):
self.__drain_conn_after_streaming_app_error()
raise gorpc.AppError(reply['Err'].get('Message',
'Missing error message'))
for field in reply['Fields']:
self._stream_fields.append((field['Name'], field['Type']))
self._stream_conversions.append(
field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return None, 0, 0, self._stream_fields
def _stream_execute(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
req = self._make_req()
req['Sql'] = sql
req['BindVariables'] = new_binds
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call('SqlQuery.StreamExecute', req)
first_response = self.client.stream_next()
reply = first_response.reply
for field in reply['Fields']:
self._stream_fields.append((field['Name'], field['Type']))
self._stream_conversions.append(field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return None, 0, 0, self._stream_fields
def _stream_execute(self, sql, bind_variables, keyspace, tablet_type, keyspace_ids=None, keyranges=None):
new_binds = field_types.convert_bind_vars(bind_variables)
exec_method = None
req = None
if keyspace_ids is not None:
req = self._create_req_with_keyspace_ids(sql, new_binds, keyspace, tablet_type, keyspace_ids)
exec_method = 'VTGate.StreamExecuteKeyspaceIds'
elif keyrange is not None:
req = self._create_req_with_keyranges(sql, new_binds, keyspace, tablet_type, keyranges)
exec_method = 'VTGate.StreamExecuteKeyRanges'
else:
raise dbexceptions.ProgrammingError('_execute called without specifying keyspace_ids or keyranges')
self._add_session(req)
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call(exec_method, req)
first_response = self.client.stream_next()
reply = first_response.reply
for field in reply['Fields']:
self._stream_fields.append((field['Name'], field['Type']))
self._stream_conversions.append(field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql, bind_variables)
except:
logging.exception('gorpc low-level error')
raise
return None, 0, 0, self._stream_fields
def _stream_execute2(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
query = self._make_req()
query["Sql"] = sql
query["BindVariables"] = new_binds
req = {"Query": query}
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call("SqlQuery.StreamExecute2", req)
first_response = self.client.stream_next()
reply = first_response.reply
if reply.get("Err"):
self.__drain_conn_after_streaming_app_error()
raise gorpc.AppError(reply["Err"].get("Message", "Missing error message"))
for field in reply["Fields"]:
self._stream_fields.append((field["Name"], field["Type"]))
self._stream_conversions.append(field_types.conversions.get(field["Type"]))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception("gorpc low-level error")
raise
return None, 0, 0, self._stream_fields
def _execute(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
req = self._make_req()
req['Sql'] = sql
req['BindVariables'] = new_binds
fields = []
conversions = []
results = []
try:
response = self.client.call('SqlQuery.Execute', req)
reply = response.reply
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def _create_v2_request_with_keyspace_ids(
sql, new_binds, keyspace_name, tablet_type, keyspace_ids,
not_in_transaction):
"""Make a request dict from arguments.
Args:
sql: Str sql with format tokens.
new_binds: Dict of bind variables.
keyspace_name: Str keyspace name.
tablet_type: Str tablet_type.
keyspace_ids: Bytes list of keyspace IDs.
not_in_transaction: Bool True if a transaction should not be started
(generally used when sql is not a write).
Returns:
A (str: value) dict.
"""
# keyspace_ids are Keyspace Ids packed to byte[]
sql, new_binds = dbapi.prepare_query_bind_vars(sql, new_binds)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': keyspace_name,
'TabletType': topodata_pb2.TabletType.Value(tablet_type.upper()),
'KeyspaceIds': keyspace_ids,
'NotInTransaction': not_in_transaction,
}
return req
def _stream_execute2(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
query = self._make_req()
query['Sql'] = sql
query['BindVariables'] = new_binds
req = {'Query': query}
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call('SqlQuery.StreamExecute2', req)
first_response = self.client.stream_next()
reply = first_response.reply
if reply.get('Err'):
self.__drain_conn_after_streaming_app_error()
raise gorpc.AppError(reply['Err'].get('Message', 'Missing error message'))
for field in reply['Fields']:
self._stream_fields.append((field['Name'], field['Type']))
self._stream_conversions.append(field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return None, 0, 0, self._stream_fields
def _stream_execute(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
req = self._make_req()
req['Sql'] = sql
req['BindVariables'] = new_binds
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call('Barnacle.StreamExecute', req)
first_response = self.client.stream_next()
reply = first_response.reply
for field in reply['Fields']:
self._stream_fields.append((field['Name'], field['Type']))
self._stream_conversions.append(
field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql, bind_variables)
except:
logging.exception('gorpc low-level error')
raise
return None, 0, 0, self._stream_fields
def _create_v2_request_with_keyranges(
sql, new_binds, keyspace_name, tablet_type, keyranges, not_in_transaction):
"""Make a request dict from arguments.
Args:
sql: Str sql with format tokens.
new_binds: Dict of bind variables.
keyspace_name: Str keyspace name.
tablet_type: Str tablet_type.
keyranges: A list of keyrange.KeyRange objects.
not_in_transaction: Bool True if a transaction should not be started
(generally used when sql is not a write).
Returns:
A (str: value) dict.
"""
sql, new_binds = dbapi.prepare_query_bind_vars(sql, new_binds)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': keyspace_name,
'TabletType': topodata_pb2.TabletType.Value(tablet_type.upper()),
'KeyRanges': keyranges,
'NotInTransaction': not_in_transaction,
}
return req
def _create_v2_request_with_shards(
sql, new_binds, keyspace_name, tablet_type, shards,
not_in_transaction):
"""Make a request dict from arguments.
Args:
sql: Str sql with format tokens.
new_binds: Dict of bind variables.
keyspace_name: Str keyspace name.
tablet_type: Str tablet_type.
shards: String list of shards.
not_in_transaction: Bool True if a transaction should not be started
(generally used when sql is not a write).
Returns:
A (str: value) dict.
"""
# keyspace_ids are Keyspace Ids packed to byte[]
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': keyspace_name,
'TabletType': topodata_pb2.TabletType.Value(tablet_type.upper()),
'Shards': shards,
'NotInTransaction': not_in_transaction,
}
return req
def _execute_batch(self, sql_list, bind_variables_list):
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
req = self._make_req()
req['Sql'] = sql
req['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query_list.append(req)
rowsets = []
try:
req = {'List': query_list}
response = self.client.call('SqlQuery.ExecuteBatch', req)
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql_list, bind_variables_list)
except:
logging.exception('gorpc low-level error')
raise
return rowsets
def _stream_execute(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': self.keyspace,
'TabletType': self.tablet_type,
'Shards': [self.shard],
}
self._add_session(req)
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call('VTGate.StreamExecuteShard', req)
first_response = self.client.stream_next()
reply = first_response.reply['Result']
for field in reply['Fields']:
self._stream_fields.append((field['Name'], field['Type']))
self._stream_conversions.append(
field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql, bind_variables)
except:
logging.exception('gorpc low-level error')
raise
return None, 0, 0, self._stream_fields
def _create_req(sql, new_binds, tablet_type):
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'TabletType': tablet_type,
}
return req
def _create_req(sql, new_binds, tablet_type):
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'TabletType': tablet_type,
}
return req
def _create_req(sql, new_binds, tablet_type, not_in_transaction):
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'TabletType': tablet_type,
'NotInTransaction': not_in_transaction,
}
return req
def _create_req(sql, new_binds, tablet_type, not_in_transaction):
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'TabletType': tablet_type,
'NotInTransaction': not_in_transaction,
}
return req
def _execute_batch(
self, sql_list, bind_variables_list, keyspace_list, keyspace_ids_list,
tablet_type, as_transaction, effective_caller_id=None):
query_list = []
for sql, bind_vars, keyspace, keyspace_ids in zip(
sql_list, bind_variables_list, keyspace_list, keyspace_ids_list):
sql, bind_vars = dbapi.prepare_query_bind_vars(sql, bind_vars)
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query['Keyspace'] = keyspace
query['KeyspaceIds'] = keyspace_ids
query_list.append(query)
rowsets = []
try:
req = {
'Queries': query_list,
'TabletType': tablet_type,
'AsTransaction': as_transaction,
}
self._add_caller_id(req, effective_caller_id)
self._add_session(req)
response = self._get_client().call('VTGate.ExecuteBatchKeyspaceIds', req)
self._update_session(response)
if response.reply.get('Error'):
raise gorpc.AppError(
response.reply['Error'], 'VTGate.ExecuteBatchKeyspaceIds')
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables_list)
raise self._convert_exception(
e, sql_list, keyspace_ids_list,
keyspace='', tablet_type=tablet_type)
except Exception:
logging.exception('gorpc low-level error')
raise
return rowsets
def _create_req_with_keyspace_ids(sql, new_binds, keyspace, tablet_type, keyspace_ids):
sql, new_binds = dbapi.prepare_query_bind_vars(sql, new_binds)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': keyspace,
'TabletType': tablet_type,
'KeyspaceIds': [str(kid) for kid in keyspace_ids],
}
return req
def _create_req_with_keyranges(sql, new_binds, keyspace, tablet_type, keyranges):
sql, new_binds = dbapi.prepare_query_bind_vars(sql, new_binds)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': keyspace,
'TabletType': tablet_type,
'KeyRanges': [keyrange.KeyRange(kr) for kr in keyranges],
}
return req
def _create_req_with_keyranges(sql, new_binds, keyspace, tablet_type, keyranges):
# keyranges are keyspace.KeyRange objects with start/end packed to byte[]
sql, new_binds = dbapi.prepare_query_bind_vars(sql, new_binds)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': keyspace,
'TabletType': tablet_type,
'KeyRanges': keyranges,
}
return req
def _create_req_with_keyranges(sql, new_binds, keyspace, tablet_type, keyranges):
# keyranges are keyspace.KeyRange objects with start/end packed to byte[]
sql, new_binds = dbapi.prepare_query_bind_vars(sql, new_binds)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': keyspace,
'TabletType': tablet_type,
'KeyRanges': keyranges,
}
return req
def _execute_entity_ids(self, sql, bind_variables, keyspace, tablet_type,
entity_keyspace_id_map, entity_column_name):
sql, new_binds = dbapi.prepare_query_bind_vars(sql, bind_variables)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql':
sql,
'BindVariables':
new_binds,
'Keyspace':
keyspace,
'TabletType':
tablet_type,
'EntityKeyspaceIDs': [{
'ExternalID': xid,
'KeyspaceID': kid
} for xid, kid in entity_keyspace_id_map.iteritems()],
'EntityColumnName':
entity_column_name,
}
self._add_session(req)
fields = []
conversions = []
results = []
try:
response = self.client.call('VTGate.ExecuteEntityIds', req)
self._update_session(response)
reply = response.reply
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'],
'VTGate.ExecuteEntityIds')
if 'Result' in reply:
res = reply['Result']
for field in res['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in res['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = res['RowsAffected']
lastrowid = res['InsertId']
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def _create_req_with_keyspace_ids(sql, new_binds, keyspace, tablet_type, keyspace_ids):
# keyspace_ids are Keyspace Ids packed to byte[]
sql, new_binds = dbapi.prepare_query_bind_vars(sql, new_binds)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': keyspace,
'TabletType': tablet_type,
'KeyspaceIds': keyspace_ids,
}
return req
def _create_req_with_keyspace_ids(sql, new_binds, keyspace, tablet_type, keyspace_ids):
# keyspace_ids are Keyspace Ids packed to byte[]
sql, new_binds = dbapi.prepare_query_bind_vars(sql, new_binds)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': keyspace,
'TabletType': tablet_type,
'KeyspaceIds': keyspace_ids,
}
return req
def _execute_entity_ids(
self, sql, bind_variables, keyspace, tablet_type,
entity_keyspace_id_map, entity_column_name, not_in_transaction=False,
effective_caller_id=None):
sql, new_binds = dbapi.prepare_query_bind_vars(sql, bind_variables)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': keyspace,
'TabletType': tablet_type,
'EntityKeyspaceIDs': [
{'ExternalID': xid, 'KeyspaceID': kid}
for xid, kid in entity_keyspace_id_map.iteritems()],
'EntityColumnName': entity_column_name,
'NotInTransaction': not_in_transaction,
}
self._add_caller_id(req, effective_caller_id)
self._add_session(req)
fields = []
conversions = []
results = []
rowcount = 0
lastrowid = 0
try:
response = self._get_client().call('VTGate.ExecuteEntityIds', req)
self._update_session(response)
reply = response.reply
if response.reply.get('Error'):
raise gorpc.AppError(response.reply['Error'], 'VTGate.ExecuteEntityIds')
if reply.get('Result'):
res = reply['Result']
for field in res['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in res['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = res['RowsAffected']
lastrowid = res['InsertId']
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise self._convert_exception(
e, sql, entity_keyspace_id_map,
keyspace=keyspace, tablet_type=tablet_type)
except Exception:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def _create_req_with_keyranges(sql, new_binds, keyspace, tablet_type, keyranges, not_in_transaction):
# keyranges are keyspace.KeyRange objects with start/end packed to byte[]
sql, new_binds = dbapi.prepare_query_bind_vars(sql, new_binds)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
"Sql": sql,
"BindVariables": new_binds,
"Keyspace": keyspace,
"TabletType": tablet_type,
"KeyRanges": keyranges,
"NotInTransaction": not_in_transaction,
}
return req
def _execute_batch(self, sql_list, bind_variables_list, as_transaction):
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query_list.append(query)
rowsets = []
try:
req = {
'QueryBatch': {
'Queries': query_list,
'SessionId': self.session_id,
'AsTransaction': as_transaction,
'TransactionId': self.transaction_id
},
'ImmediateCallerID': {
'Username': self.caller_id
}
}
response = self.rpc_call_and_extract_error(
'SqlQuery.ExecuteBatch2', req)
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables_list)
raise convert_exception(e, str(self), sql_list)
except:
logging.exception('gorpc low-level error')
raise
return rowsets
def _execute_entity_ids(self,
sql,
bind_variables,
keyspace,
tablet_type,
entity_keyspace_id_map,
entity_column_name,
not_in_transaction=False,
effective_caller_id=None):
sql, new_binds = dbapi.prepare_query_bind_vars(sql, bind_variables)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql':
sql,
'BindVariables':
new_binds,
'Keyspace':
keyspace,
'TabletType':
tablet_type,
'EntityKeyspaceIDs': [{
'ExternalID': xid,
'KeyspaceID': kid
} for xid, kid in entity_keyspace_id_map.iteritems()],
'EntityColumnName':
entity_column_name,
'NotInTransaction':
not_in_transaction,
}
self._add_caller_id(req, effective_caller_id)
self._add_session(req)
try:
exec_method = 'VTGate.ExecuteEntityIds'
response = self._get_client().call(exec_method, req)
self._update_session(response)
vtgate_utils.extract_rpc_error(exec_method, response)
reply = response.reply
return self._get_rowset_from_query_result(reply.get('Result'))
except (gorpc.GoRpcError, vtgate_utils.VitessError) as e:
self.logger_object.log_private_data(bind_variables)
raise self._convert_exception(e,
sql,
entity_keyspace_id_map,
keyspace=keyspace,
tablet_type=tablet_type)
except Exception:
logging.exception('gorpc low-level error')
raise
def _execute_batch(self, sql_list, bind_variables_list):
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query_list.append(query)
rowsets = []
try:
req = {
'Queries': query_list,
'Keyspace': self.keyspace,
'TabletType': self.tablet_type,
'Shards': [self.shard],
}
self._add_session(req)
response = self.client.call('VTGate.ExecuteBatchShard', req)
self._update_session(response)
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'],
'VTGate.ExecuteBatchShard')
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql_list,
bind_variables_list)
except:
logging.exception('gorpc low-level error')
raise
return rowsets
def _execute_batch(self, sql_list, bind_variables_list, tablet_type,
as_transaction):
"""Send a list of queries and variables to VTGate.ExecuteBatch."""
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query_list.append(query)
rowsets = []
try:
req = {
'Queries': query_list,
'TabletType': tablet_type,
'AsTransaction': as_transaction,
}
self._add_session(req)
response = self._get_client().call('VTGate.ExecuteBatch', req)
self._update_session(response)
vtgate_utils.extract_rpc_error('VTGate.ExecuteBatch', response)
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except (gorpc.GoRpcError, vtgate_utils.VitessError) as e:
self.logger_object.log_private_data(bind_variables_list)
raise convert_exception(e, str(self), sql_list)
except Exception:
logging.exception('gorpc low-level error')
raise
return rowsets
def _execute_batch(self, sql_list, bind_variables_list, as_transaction):
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query_list.append(query)
rowsets = []
try:
req = {
'QueryBatch': {
'Queries': query_list,
'SessionId': self.session_id,
'AsTransaction': as_transaction,
'TransactionId': self.transaction_id
},
'ImmediateCallerID': {'Username': self.caller_id}
}
response = self.rpc_call_and_extract_error('SqlQuery.ExecuteBatch2', req)
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables_list)
raise convert_exception(e, str(self), sql_list)
except Exception:
logging.exception('gorpc low-level error')
raise
return rowsets
def _execute_batch(self, sql_list, bind_variables_list, keyspace, tablet_type, keyspace_ids):
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
sql, bind_vars = dbapi.prepare_query_bind_vars(sql, bind_vars)
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query_list.append(query)
rowsets = []
try:
req = {
'Queries': query_list,
'Keyspace': keyspace,
'TabletType': tablet_type,
'KeyspaceIds': keyspace_ids,
}
self._add_session(req)
response = self.client.call('VTGate.ExecuteBatchKeyspaceIds', req)
self._update_session(response)
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'], 'VTGate.ExecuteBatchKeyspaceIds')
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql_list, bind_variables_list)
except:
logging.exception('gorpc low-level error')
raise
return rowsets
def _execute_batch(
self, sql_list, bind_variables_list, tablet_type, as_transaction):
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query_list.append(query)
rowsets = []
try:
req = {
'Queries': query_list,
'TabletType': tablet_type,
'AsTransaction': as_transaction,
}
self._add_session(req)
response = self._get_client().call('VTGate.ExecuteBatch', req)
self._update_session(response)
vtgate_utils.extract_rpc_error('VTGate.ExecuteBatch', response)
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except (gorpc.GoRpcError, vtgate_utils.VitessError) as e:
self.logger_object.log_private_data(bind_variables_list)
raise convert_exception(e, str(self), sql_list)
except Exception:
logging.exception('gorpc low-level error')
raise
return rowsets
def _execute_entity_ids(self, sql, bind_variables, keyspace, tablet_type, entity_keyspace_id_map, entity_column_name):
sql, new_binds = dbapi.prepare_query_bind_vars(sql, bind_variables)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': keyspace,
'TabletType': tablet_type,
'EntityKeyspaceIDs': [
{'ExternalID': xid, 'KeyspaceID': kid}
for xid, kid in entity_keyspace_id_map.iteritems()],
'EntityColumnName': entity_column_name,
}
self._add_session(req)
fields = []
conversions = []
results = []
try:
response = self.client.call('VTGate.ExecuteEntityIds', req)
self._update_session(response)
reply = response.reply
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'], 'VTGate.ExecuteEntityIds')
if 'Result' in reply:
res = reply['Result']
for field in res['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in res['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = res['RowsAffected']
lastrowid = res['InsertId']
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql, bind_variables)
except:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def _execute(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': self.keyspace,
'TabletType': self.tablet_type,
'Shards': [self.shard],
}
self._add_session(req)
fields = []
conversions = []
results = []
rowcount = 0
lastrowid = 0
try:
response = self.client.call('VTGate.ExecuteShard', req)
self._update_session(response)
reply = response.reply
# TODO(sougou): Simplify this check after all servers are deployed
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'],
'VTGate.ExecuteShard')
if 'Result' in reply:
res = reply['Result']
for field in res['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in res['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = res['RowsAffected']
lastrowid = res['InsertId']
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql, bind_variables)
except:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def build_query_list():
"""Create a query dict list from parameters."""
query_list = []
for sql, bind_vars, keyspace_name, keyspace_ids, shards in zip(
sql_list, bind_variables_list, keyspace_list, keyspace_ids_list,
shards_list):
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query['Keyspace'] = keyspace_name
if keyspace_ids:
if shards:
raise dbexceptions.ProgrammingError(
'Keyspace_ids and shards cannot both be defined '
'for the same executemany query.')
query['KeyspaceIds'] = keyspace_ids
else:
query['Shards'] = shards
query_list.append(query)
return query_list
def build_query_list():
query_list = []
for sql, bind_vars, keyspace, keyspace_ids, shards in zip(
sql_list, bind_variables_list, keyspace_list, keyspace_ids_list, shards_list
):
sql, bind_vars = dbapi.prepare_query_bind_vars(sql, bind_vars)
query = {}
query["Sql"] = sql
query["BindVariables"] = field_types.convert_bind_vars(bind_vars)
query["Keyspace"] = keyspace
if keyspace_ids:
if shards:
raise dbexceptions.ProgrammingError(
"Keyspace_ids and shards cannot both be defined " "for the same executemany query."
)
query["KeyspaceIds"] = keyspace_ids
else:
query["Shards"] = shards
query_list.append(query)
return query_list
def _execute(self, sql, bind_variables, keyspace, tablet_type, keyspace_ids=None, keyranges=None):
new_binds = field_types.convert_bind_vars(bind_variables)
exec_method = None
req = None
if keyspace_ids is not None:
req = self._create_req_with_keyspace_ids(sql, new_binds, keyspace, tablet_type, keyspace_ids)
exec_method = 'VTGate.ExecuteKeyspaceIds'
elif keyrange is not None:
req = self._create_req_with_keyranges(sql, new_binds, keyspace, tablet_type, keyranges)
exec_method = 'VTGate.ExecuteKeyRanges'
else:
raise dbexceptions.ProgrammingError('_execute called without specifying keyspace_ids or keyranges')
self._add_session(req)
fields = []
conversions = []
results = []
try:
response = self.client.call(exec_method, req)
self._update_session(response)
reply = response.reply
if 'Error' in response.reply:
raise gorpc.AppError(response.reply['Error'], exec_method)
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql, bind_variables)
except:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def _execute_batch(self, sql_list, bind_variables_list, tablet_type):
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
sql, bind_vars = dbapi.prepare_query_bind_vars(sql, bind_vars)
query = {}
query["Sql"] = sql
query["BindVariables"] = field_types.convert_bind_vars(bind_vars)
query_list.append(query)
rowsets = []
try:
req = {"Queries": query_list, "TabletType": tablet_type}
self._add_session(req)
response = self.client.call("VTGate.ExecuteBatch", req)
self._update_session(response)
if "Error" in response.reply and response.reply["Error"]:
raise gorpc.AppError(response.reply["Error"], "VTGate.ExecuteBatch")
for reply in response.reply["List"]:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply["Fields"]:
fields.append((field["Name"], field["Type"]))
conversions.append(field_types.conversions.get(field["Type"]))
for row in reply["Rows"]:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply["RowsAffected"]
lastrowid = reply["InsertId"]
rowsets.append((results, rowcount, lastrowid, fields))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables_list)
raise convert_exception(e, str(self), sql_list)
except:
logging.exception("gorpc low-level error")
raise
return rowsets
def build_query_list():
query_list = []
for sql, bind_vars, keyspace, keyspace_ids, shards in zip(
sql_list, bind_variables_list, keyspace_list,
keyspace_ids_list, shards_list):
sql, bind_vars = dbapi.prepare_query_bind_vars(sql, bind_vars)
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(
bind_vars)
query['Keyspace'] = keyspace
if keyspace_ids:
if shards:
raise dbexceptions.ProgrammingError(
'Keyspace_ids and shards cannot both be defined '
'for the same executemany query.')
query['KeyspaceIds'] = keyspace_ids
else:
query['Shards'] = shards
query_list.append(query)
return query_list
def _create_v3_request(sql, new_binds, tablet_type, not_in_transaction):
"""Make a request where routing info is derived from sql and bind vars.
Args:
sql: Str sql with format tokens.
new_binds: Dict of bind variables.
tablet_type: Str tablet_type.
not_in_transaction: Bool True if a transaction should not be started
(generally used when sql is not a write).
Returns:
A (str: value) dict.
"""
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql': sql,
'BindVariables': new_binds,
'TabletType': topodata_pb2.TabletType.Value(tablet_type.upper()),
'NotInTransaction': not_in_transaction,
}
return req
def _execute_batch(self, sql_list, bind_variables_list):
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query_list.append(query)
rowsets = []
try:
req = self._make_req()
req['Queries'] = query_list
response = self.client.call('SqlQuery.ExecuteBatch', req)
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables_list)
raise convert_exception(e, str(self), sql_list)
except:
logging.exception('gorpc low-level error')
raise
return rowsets
def _stream_execute(self, sql, bind_variables):
req = {
'Query': {
'Sql': sql,
'BindVariables': field_types.convert_bind_vars(bind_variables),
'SessionId': self.session_id,
'TransactionId': self.transaction_id
},
'ImmediateCallerID': {
'Username': self.caller_id
}
}
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call('SqlQuery.StreamExecute2', req)
first_response = self.client.stream_next()
reply = first_response.reply
if reply.get('Err'):
self.__drain_conn_after_streaming_app_error()
raise gorpc.AppError(reply['Err'].get('Message',
'Missing error message'))
for field in reply['Fields']:
self._stream_fields.append((field['Name'], field['Type']))
self._stream_conversions.append(
field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return None, 0, 0, self._stream_fields
def _execute(self,
sql,
bind_variables,
keyspace_name,
tablet_type,
keyspace_ids=None,
keyranges=None,
entity_keyspace_id_map=None,
entity_column_name=None,
not_in_transaction=False,
effective_caller_id=None):
"""Execute query.
Args:
sql: The sql text, with %(format)s-style tokens.
bind_variables: (str: value) dict of bind variables corresponding
to sql %(format)s tokens.
keyspace_name: Str name of keyspace.
tablet_type: Str tablet type (e.g. master, rdonly, replica).
keyspace_ids: bytes list of keyspace ID lists.
keyranges: KeyRange objects.
entity_keyspace_id_map: (column value: bytes) map from a column
to a keyspace id. If defined, vtgate adds a per-shard expression
to the WHERE clause, and ignores keyspace_ids and keyranges
parameters.
entity_column_name: Str name of entity column used by
entity_keyspace_id_map.
not_in_transaction: bool.
effective_caller_id: CallerID object.
Returns:
The (results, rowcount, lastrowid, fields) tuple.
"""
routing_kwargs = {}
exec_method = None
req = None
if entity_keyspace_id_map is not None:
# This supercedes keyspace_ids and keyranges.
routing_kwargs['entity_keyspace_id_map'] = entity_keyspace_id_map
routing_kwargs['entity_column_name'] = entity_column_name
if entity_column_name is None:
raise dbexceptions.ProgrammingError(
'_execute called with entity_keyspace_id_map and no '
'entity_column_name')
sql, new_binds = dbapi.prepare_query_bind_vars(sql, bind_variables)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql':
sql,
'BindVariables':
new_binds,
'Keyspace':
keyspace_name,
'TabletType':
tablet_type,
'EntityKeyspaceIDs': [{
'ExternalID': xid,
'KeyspaceID': kid
} for xid, kid in entity_keyspace_id_map.iteritems()],
'EntityColumnName':
entity_column_name,
'NotInTransaction':
not_in_transaction,
}
exec_method = 'VTGate.ExecuteEntityIds'
elif keyspace_ids is not None:
if keyranges is not None:
raise dbexceptions.ProgrammingError(
'_execute called with keyspace_ids and keyranges both defined'
)
routing_kwargs['keyspace_ids'] = keyspace_ids
req = _create_req_with_keyspace_ids(sql, bind_variables,
keyspace_name, tablet_type,
keyspace_ids,
not_in_transaction)
exec_method = 'VTGate.ExecuteKeyspaceIds'
elif keyranges is not None:
routing_kwargs['keyranges'] = keyranges
req = _create_req_with_keyranges(sql, bind_variables,
keyspace_name, tablet_type,
keyranges, not_in_transaction)
exec_method = 'VTGate.ExecuteKeyRanges'
else:
raise dbexceptions.ProgrammingError(
'_execute called with no entity_keyspace_id_map, '
'keyspace_ids, or keyranges')
self._add_caller_id(req, effective_caller_id)
if not not_in_transaction:
self._add_session(req)
try:
response = self._get_client().call(exec_method, req)
self._update_session(response)
vtgate_utils.extract_rpc_error(exec_method, response)
reply = response.reply
return self._get_rowset_from_query_result(reply.get('Result'))
except (gorpc.GoRpcError, vtgate_utils.VitessError) as e:
self.logger_object.log_private_data(bind_variables)
raise self._convert_exception(e,
sql,
keyspace=keyspace_name,
tablet_type=tablet_type,
**routing_kwargs)
except Exception:
logging.exception('gorpc low-level error')
raise
def _stream_execute(self, sql, bind_variables):
req = {
'Query': {
'Sql': sql,
'BindVariables': field_types.convert_bind_vars(bind_variables),
'SessionId': self.session_id,
'TransactionId': self.transaction_id
},
'ImmediateCallerID': {
'Username': self.caller_id
}
}
rpc_client = self._get_client()
stream_fields = []
stream_conversions = []
def drain_conn_after_streaming_app_error():
"""Drain connection of all incoming streaming packets (ignoring them).
This is necessary for streaming calls which return application
errors inside the RPC response (instead of through the usual GoRPC
error return). This is because GoRPC always expects the last
packet to be an error; either the usual GoRPC application error
return, or a special "end-of-stream" error.
If an application error is returned with the RPC response, there
will still be at least one more packet coming, as GoRPC has not
seen anything that it considers to be an error. If the connection
is not drained of this last packet, future reads from the wire
will be off by one and will return errors.
"""
next_result = rpc_client.stream_next()
if next_result is not None:
rpc_client.close()
raise gorpc.GoRpcError(
'Connection should only have one packet remaining'
' after streaming app error in RPC response.')
try:
rpc_client.stream_call('SqlQuery.StreamExecute2', req)
first_response = rpc_client.stream_next()
reply = first_response.reply
if reply.get('Err'):
drain_conn_after_streaming_app_error()
raise gorpc.AppError(reply['Err'].get('Message',
'Missing error message'))
for field in reply['Fields']:
stream_fields.append((field['Name'], field['Type']))
stream_conversions.append(
field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except Exception:
logging.exception('gorpc low-level error')
raise
# Take the BsonRpcClient from VTGateConnection. The row_generator
# will manage the BsonRpcClient. This VTGateConnection will connect
# to a new client if needed.
self.client = None
def row_generator():
try:
while True:
try:
stream_result = rpc_client.stream_next()
if stream_result is None:
break
if stream_result.reply.get('Err'):
drain_conn_after_streaming_app_error()
raise gorpc.AppError(
stream_result.reply['Err'].get(
'Message', 'Missing error message'))
for result_item in stream_result.reply['Rows']:
yield tuple(
_make_row(result_item, stream_conversions))
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self))
except Exception:
logging.exception('gorpc low-level error')
raise
finally:
rpc_client.close()
return row_generator(), stream_fields
