def _execute(self, sql, bind_variables, tablet_type):
req = _create_req(sql, bind_variables, tablet_type)
self._add_session(req)
fields = []
conversions = []
results = []
rowcount = 0
lastrowid = 0
try:
response = self.client.call('VTGate.Execute', req)
self._update_session(response)
reply = response.reply
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'], 'VTGate.Execute')
if 'Result' in reply:
res = reply['Result']
for field in res['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in res['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = res['RowsAffected']
lastrowid = res['InsertId']
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def _stream_next(self):
# Terminating condition
if self._stream_result_index is None:
return None
# See if we need to read more or whether we just pop the next row.
if self._stream_result is None:
try:
self._stream_result = self.client.stream_next()
if self._stream_result is None:
self._stream_result_index = None
return None
if self._stream_result.reply.get('Err'):
self.__drain_conn_after_streaming_app_error()
raise gorpc.AppError(self._stream_result.reply['Err'].get(
'Message', 'Missing error message'))
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self))
except:
logging.exception('gorpc low-level error')
raise
row = tuple(
_make_row(
self._stream_result.reply['Rows'][self._stream_result_index],
self._stream_conversions))
# If we are reading the last row, set us up to read more data.
self._stream_result_index += 1
if self._stream_result_index == len(self._stream_result.reply['Rows']):
self._stream_result = None
self._stream_result_index = 0
return row
def _execute_batch(self,
sql_list,
bind_variables_list,
keyspace,
tablet_type,
keyspace_ids,
not_in_transaction=False):
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
sql, bind_vars = dbapi.prepare_query_bind_vars(sql, bind_vars)
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query_list.append(query)
rowsets = []
try:
req = {
'Queries': query_list,
'Keyspace': keyspace,
'TabletType': tablet_type,
'KeyspaceIds': keyspace_ids,
'NotInTransaction': not_in_transaction,
}
self._add_session(req)
response = self.client.call('VTGate.ExecuteBatchKeyspaceIds', req)
self._update_session(response)
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'],
'VTGate.ExecuteBatchKeyspaceIds')
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables_list)
raise convert_exception(e,
str(self),
sql_list,
keyspace_ids,
keyspace=keyspace,
tablet_type=tablet_type)
except:
logging.exception('gorpc low-level error')
raise
return rowsets
def _stream_execute2(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
query = self._make_req()
query['Sql'] = sql
query['BindVariables'] = new_binds
req = {'Query': query}
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call('SqlQuery.StreamExecute2', req)
first_response = self.client.stream_next()
reply = first_response.reply
if reply.get('Err'):
self.__drain_conn_after_streaming_app_error()
raise gorpc.AppError(reply['Err'].get('Message',
'Missing error message'))
for field in reply['Fields']:
self._stream_fields.append((field['Name'], field['Type']))
self._stream_conversions.append(
field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return None, 0, 0, self._stream_fields
def _execute(self,
sql,
bind_variables,
keyspace,
tablet_type,
keyspace_ids=None,
keyranges=None):
exec_method = None
req = None
if keyspace_ids is not None:
req = _create_req_with_keyspace_ids(sql, bind_variables, keyspace,
tablet_type, keyspace_ids)
exec_method = 'VTGate.ExecuteKeyspaceIds'
elif keyranges is not None:
req = _create_req_with_keyranges(sql, bind_variables, keyspace,
tablet_type, keyranges)
exec_method = 'VTGate.ExecuteKeyRanges'
else:
raise dbexceptions.ProgrammingError(
'_execute called without specifying keyspace_ids or keyranges')
self._add_session(req)
fields = []
conversions = []
results = []
rowcount = 0
lastrowid = 0
try:
response = self.client.call(exec_method, req)
self._update_session(response)
reply = response.reply
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'], exec_method)
if 'Result' in reply:
res = reply['Result']
for field in res['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in res['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = res['RowsAffected']
lastrowid = res['InsertId']
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def _execute_entity_ids(self, sql, bind_variables, keyspace, tablet_type,
entity_keyspace_id_map, entity_column_name):
sql, new_binds = dbapi.prepare_query_bind_vars(sql, bind_variables)
new_binds = field_types.convert_bind_vars(new_binds)
req = {
'Sql':
sql,
'BindVariables':
new_binds,
'Keyspace':
keyspace,
'TabletType':
tablet_type,
'EntityKeyspaceIDs': [{
'ExternalID': xid,
'KeyspaceID': kid
} for xid, kid in entity_keyspace_id_map.iteritems()],
'EntityColumnName':
entity_column_name,
}
self._add_session(req)
fields = []
conversions = []
results = []
try:
response = self.client.call('VTGate.ExecuteEntityIds', req)
self._update_session(response)
reply = response.reply
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'],
'VTGate.ExecuteEntityIds')
if 'Result' in reply:
res = reply['Result']
for field in res['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in res['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = res['RowsAffected']
lastrowid = res['InsertId']
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def rpc_call_and_extract_error(self, method_name, request):
"""Makes an RPC call, and extracts any app error that's embedded in the reply.
Args:
method_name - RPC method name, as a string, to call
request - request to send to the RPC method call
Raises:
gorpc.AppError if there is an app error embedded in the reply
"""
response = self.client.call(method_name, request)
reply = response.reply
if not reply or not isinstance(reply, dict):
return response
# Handle the case of new client => old server
err = reply.get('Err', None)
if err:
if not isinstance(reply, dict) or 'Message' not in err:
raise gorpc.AppError('Missing error message', method_name)
raise gorpc.AppError(reply['Err']['Message'], method_name)
return response
def _execute_batch(self, sql_list, bind_variables_list):
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query_list.append(query)
rowsets = []
try:
req = {
'Queries': query_list,
'Keyspace': self.keyspace,
'TabletType': self.tablet_type,
'Shards': [self.shard],
}
self._add_session(req)
response = self.client.call('VTGate.ExecuteBatchShard', req)
self._update_session(response)
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'],
'VTGate.ExecuteBatchShard')
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql_list,
bind_variables_list)
except:
logging.exception('gorpc low-level error')
raise
return rowsets
def _execute_batch(
self, sql_list, bind_variables_list, tablet_type, as_transaction):
query_list = []
for sql, bind_vars in zip(sql_list, bind_variables_list):
query = {}
query['Sql'] = sql
query['BindVariables'] = field_types.convert_bind_vars(bind_vars)
query_list.append(query)
rowsets = []
try:
req = {
'Queries': query_list,
'TabletType': tablet_type,
'AsTransaction': as_transaction,
}
self._add_session(req)
response = self._get_client().call('VTGate.ExecuteBatch', req)
self._update_session(response)
if response.reply.get('Error'):
raise gorpc.AppError(response.reply['Error'], 'VTGate.ExecuteBatch')
for reply in response.reply['List']:
fields = []
conversions = []
results = []
rowcount = 0
for field in reply['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(field_types.conversions.get(field['Type']))
for row in reply['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = reply['RowsAffected']
lastrowid = reply['InsertId']
rowsets.append((results, rowcount, lastrowid, fields))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables_list)
raise convert_exception(e, str(self), sql_list)
except Exception:
logging.exception('gorpc low-level error')
raise
return rowsets
def _execute(self, sql, bind_variables):
new_binds = field_types.convert_bind_vars(bind_variables)
req = {
'Sql': sql,
'BindVariables': new_binds,
'Keyspace': self.keyspace,
'TabletType': self.tablet_type,
'Shards': [self.shard],
}
self._add_session(req)
fields = []
conversions = []
results = []
rowcount = 0
lastrowid = 0
try:
response = self.client.call('VTGate.ExecuteShard', req)
self._update_session(response)
reply = response.reply
# TODO(sougou): Simplify this check after all servers are deployed
if 'Error' in response.reply and response.reply['Error']:
raise gorpc.AppError(response.reply['Error'],
'VTGate.ExecuteShard')
if 'Result' in reply:
res = reply['Result']
for field in res['Fields']:
fields.append((field['Name'], field['Type']))
conversions.append(
field_types.conversions.get(field['Type']))
for row in res['Rows']:
results.append(tuple(_make_row(row, conversions)))
rowcount = res['RowsAffected']
lastrowid = res['InsertId']
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self), sql, bind_variables)
except:
logging.exception('gorpc low-level error')
raise
return results, rowcount, lastrowid, fields
def row_generator():
try:
while True:
try:
stream_result = rpc_client.stream_next()
if stream_result is None:
break
if stream_result.reply.get('Err'):
drain_conn_after_streaming_app_error()
raise gorpc.AppError(
stream_result.reply['Err'].get(
'Message', 'Missing error message'))
for result_item in stream_result.reply['Rows']:
yield tuple(
_make_row(result_item, stream_conversions))
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self))
except Exception:
logging.exception('gorpc low-level error')
raise
finally:
rpc_client.close()
def _stream_execute(self, sql, bind_variables):
req = {
'Query': {
'Sql': sql,
'BindVariables': field_types.convert_bind_vars(bind_variables),
'SessionId': self.session_id,
'TransactionId': self.transaction_id
},
'ImmediateCallerID': {
'Username': self.caller_id
}
}
self._stream_fields = []
self._stream_conversions = []
self._stream_result = None
self._stream_result_index = 0
try:
self.client.stream_call('SqlQuery.StreamExecute2', req)
first_response = self.client.stream_next()
reply = first_response.reply
if reply.get('Err'):
self.__drain_conn_after_streaming_app_error()
raise gorpc.AppError(reply['Err'].get('Message',
'Missing error message'))
for field in reply['Fields']:
self._stream_fields.append((field['Name'], field['Type']))
self._stream_conversions.append(
field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except:
logging.exception('gorpc low-level error')
raise
return None, 0, 0, self._stream_fields
def _stream_execute(self, sql, bind_variables):
req = {
'Query': {
'Sql': sql,
'BindVariables': field_types.convert_bind_vars(bind_variables),
'SessionId': self.session_id,
'TransactionId': self.transaction_id
},
'ImmediateCallerID': {
'Username': self.caller_id
}
}
rpc_client = self._get_client()
stream_fields = []
stream_conversions = []
def drain_conn_after_streaming_app_error():
"""Drain connection of all incoming streaming packets (ignoring them).
This is necessary for streaming calls which return application
errors inside the RPC response (instead of through the usual GoRPC
error return). This is because GoRPC always expects the last
packet to be an error; either the usual GoRPC application error
return, or a special "end-of-stream" error.
If an application error is returned with the RPC response, there
will still be at least one more packet coming, as GoRPC has not
seen anything that it considers to be an error. If the connection
is not drained of this last packet, future reads from the wire
will be off by one and will return errors.
"""
next_result = rpc_client.stream_next()
if next_result is not None:
rpc_client.close()
raise gorpc.GoRpcError(
'Connection should only have one packet remaining'
' after streaming app error in RPC response.')
try:
rpc_client.stream_call('SqlQuery.StreamExecute2', req)
first_response = rpc_client.stream_next()
reply = first_response.reply
if reply.get('Err'):
drain_conn_after_streaming_app_error()
raise gorpc.AppError(reply['Err'].get('Message',
'Missing error message'))
for field in reply['Fields']:
stream_fields.append((field['Name'], field['Type']))
stream_conversions.append(
field_types.conversions.get(field['Type']))
except gorpc.GoRpcError as e:
self.logger_object.log_private_data(bind_variables)
raise convert_exception(e, str(self), sql)
except Exception:
logging.exception('gorpc low-level error')
raise
# Take the BsonRpcClient from VTGateConnection. The row_generator
# will manage the BsonRpcClient. This VTGateConnection will connect
# to a new client if needed.
self.client = None
def row_generator():
try:
while True:
try:
stream_result = rpc_client.stream_next()
if stream_result is None:
break
if stream_result.reply.get('Err'):
drain_conn_after_streaming_app_error()
raise gorpc.AppError(
stream_result.reply['Err'].get(
'Message', 'Missing error message'))
for result_item in stream_result.reply['Rows']:
yield tuple(
_make_row(result_item, stream_conversions))
except gorpc.GoRpcError as e:
raise convert_exception(e, str(self))
except Exception:
logging.exception('gorpc low-level error')
raise
finally:
rpc_client.close()
return row_generator(), stream_fields
