def _execute(self, sql, bind_variables, tablet_type): req = _create_req(sql, bind_variables, tablet_type) self._add_session(req) fields = [] conversions = [] results = [] rowcount = 0 lastrowid = 0 try: response = self.client.call('VTGate.Execute', req) self._update_session(response) reply = response.reply if 'Error' in response.reply and response.reply['Error']: raise gorpc.AppError(response.reply['Error'], 'VTGate.Execute') if 'Result' in reply: res = reply['Result'] for field in res['Fields']: fields.append((field['Name'], field['Type'])) conversions.append( field_types.conversions.get(field['Type'])) for row in res['Rows']: results.append(tuple(_make_row(row, conversions))) rowcount = res['RowsAffected'] lastrowid = res['InsertId'] except gorpc.GoRpcError as e: self.logger_object.log_private_data(bind_variables) raise convert_exception(e, str(self), sql) except: logging.exception('gorpc low-level error') raise return results, rowcount, lastrowid, fields
def _stream_next(self): # Terminating condition if self._stream_result_index is None: return None # See if we need to read more or whether we just pop the next row. if self._stream_result is None: try: self._stream_result = self.client.stream_next() if self._stream_result is None: self._stream_result_index = None return None if self._stream_result.reply.get('Err'): self.__drain_conn_after_streaming_app_error() raise gorpc.AppError(self._stream_result.reply['Err'].get( 'Message', 'Missing error message')) except gorpc.GoRpcError as e: raise convert_exception(e, str(self)) except: logging.exception('gorpc low-level error') raise row = tuple( _make_row( self._stream_result.reply['Rows'][self._stream_result_index], self._stream_conversions)) # If we are reading the last row, set us up to read more data. self._stream_result_index += 1 if self._stream_result_index == len(self._stream_result.reply['Rows']): self._stream_result = None self._stream_result_index = 0 return row
def _execute_batch(self, sql_list, bind_variables_list, keyspace, tablet_type, keyspace_ids, not_in_transaction=False): query_list = [] for sql, bind_vars in zip(sql_list, bind_variables_list): sql, bind_vars = dbapi.prepare_query_bind_vars(sql, bind_vars) query = {} query['Sql'] = sql query['BindVariables'] = field_types.convert_bind_vars(bind_vars) query_list.append(query) rowsets = [] try: req = { 'Queries': query_list, 'Keyspace': keyspace, 'TabletType': tablet_type, 'KeyspaceIds': keyspace_ids, 'NotInTransaction': not_in_transaction, } self._add_session(req) response = self.client.call('VTGate.ExecuteBatchKeyspaceIds', req) self._update_session(response) if 'Error' in response.reply and response.reply['Error']: raise gorpc.AppError(response.reply['Error'], 'VTGate.ExecuteBatchKeyspaceIds') for reply in response.reply['List']: fields = [] conversions = [] results = [] rowcount = 0 for field in reply['Fields']: fields.append((field['Name'], field['Type'])) conversions.append( field_types.conversions.get(field['Type'])) for row in reply['Rows']: results.append(tuple(_make_row(row, conversions))) rowcount = reply['RowsAffected'] lastrowid = reply['InsertId'] rowsets.append((results, rowcount, lastrowid, fields)) except gorpc.GoRpcError as e: self.logger_object.log_private_data(bind_variables_list) raise convert_exception(e, str(self), sql_list, keyspace_ids, keyspace=keyspace, tablet_type=tablet_type) except: logging.exception('gorpc low-level error') raise return rowsets
def _stream_execute2(self, sql, bind_variables): new_binds = field_types.convert_bind_vars(bind_variables) query = self._make_req() query['Sql'] = sql query['BindVariables'] = new_binds req = {'Query': query} self._stream_fields = [] self._stream_conversions = [] self._stream_result = None self._stream_result_index = 0 try: self.client.stream_call('SqlQuery.StreamExecute2', req) first_response = self.client.stream_next() reply = first_response.reply if reply.get('Err'): self.__drain_conn_after_streaming_app_error() raise gorpc.AppError(reply['Err'].get('Message', 'Missing error message')) for field in reply['Fields']: self._stream_fields.append((field['Name'], field['Type'])) self._stream_conversions.append( field_types.conversions.get(field['Type'])) except gorpc.GoRpcError as e: self.logger_object.log_private_data(bind_variables) raise convert_exception(e, str(self), sql) except: logging.exception('gorpc low-level error') raise return None, 0, 0, self._stream_fields
def _execute(self, sql, bind_variables, keyspace, tablet_type, keyspace_ids=None, keyranges=None): exec_method = None req = None if keyspace_ids is not None: req = _create_req_with_keyspace_ids(sql, bind_variables, keyspace, tablet_type, keyspace_ids) exec_method = 'VTGate.ExecuteKeyspaceIds' elif keyranges is not None: req = _create_req_with_keyranges(sql, bind_variables, keyspace, tablet_type, keyranges) exec_method = 'VTGate.ExecuteKeyRanges' else: raise dbexceptions.ProgrammingError( '_execute called without specifying keyspace_ids or keyranges') self._add_session(req) fields = [] conversions = [] results = [] rowcount = 0 lastrowid = 0 try: response = self.client.call(exec_method, req) self._update_session(response) reply = response.reply if 'Error' in response.reply and response.reply['Error']: raise gorpc.AppError(response.reply['Error'], exec_method) if 'Result' in reply: res = reply['Result'] for field in res['Fields']: fields.append((field['Name'], field['Type'])) conversions.append( field_types.conversions.get(field['Type'])) for row in res['Rows']: results.append(tuple(_make_row(row, conversions))) rowcount = res['RowsAffected'] lastrowid = res['InsertId'] except gorpc.GoRpcError as e: self.logger_object.log_private_data(bind_variables) raise convert_exception(e, str(self), sql) except: logging.exception('gorpc low-level error') raise return results, rowcount, lastrowid, fields
def _execute_entity_ids(self, sql, bind_variables, keyspace, tablet_type, entity_keyspace_id_map, entity_column_name): sql, new_binds = dbapi.prepare_query_bind_vars(sql, bind_variables) new_binds = field_types.convert_bind_vars(new_binds) req = { 'Sql': sql, 'BindVariables': new_binds, 'Keyspace': keyspace, 'TabletType': tablet_type, 'EntityKeyspaceIDs': [{ 'ExternalID': xid, 'KeyspaceID': kid } for xid, kid in entity_keyspace_id_map.iteritems()], 'EntityColumnName': entity_column_name, } self._add_session(req) fields = [] conversions = [] results = [] try: response = self.client.call('VTGate.ExecuteEntityIds', req) self._update_session(response) reply = response.reply if 'Error' in response.reply and response.reply['Error']: raise gorpc.AppError(response.reply['Error'], 'VTGate.ExecuteEntityIds') if 'Result' in reply: res = reply['Result'] for field in res['Fields']: fields.append((field['Name'], field['Type'])) conversions.append( field_types.conversions.get(field['Type'])) for row in res['Rows']: results.append(tuple(_make_row(row, conversions))) rowcount = res['RowsAffected'] lastrowid = res['InsertId'] except gorpc.GoRpcError as e: self.logger_object.log_private_data(bind_variables) raise convert_exception(e, str(self), sql) except: logging.exception('gorpc low-level error') raise return results, rowcount, lastrowid, fields
def rpc_call_and_extract_error(self, method_name, request): """Makes an RPC call, and extracts any app error that's embedded in the reply. Args: method_name - RPC method name, as a string, to call request - request to send to the RPC method call Raises: gorpc.AppError if there is an app error embedded in the reply """ response = self.client.call(method_name, request) reply = response.reply if not reply or not isinstance(reply, dict): return response # Handle the case of new client => old server err = reply.get('Err', None) if err: if not isinstance(reply, dict) or 'Message' not in err: raise gorpc.AppError('Missing error message', method_name) raise gorpc.AppError(reply['Err']['Message'], method_name) return response
def _execute_batch(self, sql_list, bind_variables_list): query_list = [] for sql, bind_vars in zip(sql_list, bind_variables_list): query = {} query['Sql'] = sql query['BindVariables'] = field_types.convert_bind_vars(bind_vars) query_list.append(query) rowsets = [] try: req = { 'Queries': query_list, 'Keyspace': self.keyspace, 'TabletType': self.tablet_type, 'Shards': [self.shard], } self._add_session(req) response = self.client.call('VTGate.ExecuteBatchShard', req) self._update_session(response) if 'Error' in response.reply and response.reply['Error']: raise gorpc.AppError(response.reply['Error'], 'VTGate.ExecuteBatchShard') for reply in response.reply['List']: fields = [] conversions = [] results = [] rowcount = 0 for field in reply['Fields']: fields.append((field['Name'], field['Type'])) conversions.append( field_types.conversions.get(field['Type'])) for row in reply['Rows']: results.append(tuple(_make_row(row, conversions))) rowcount = reply['RowsAffected'] lastrowid = reply['InsertId'] rowsets.append((results, rowcount, lastrowid, fields)) except gorpc.GoRpcError as e: raise convert_exception(e, str(self), sql_list, bind_variables_list) except: logging.exception('gorpc low-level error') raise return rowsets
def _execute_batch( self, sql_list, bind_variables_list, tablet_type, as_transaction): query_list = [] for sql, bind_vars in zip(sql_list, bind_variables_list): query = {} query['Sql'] = sql query['BindVariables'] = field_types.convert_bind_vars(bind_vars) query_list.append(query) rowsets = [] try: req = { 'Queries': query_list, 'TabletType': tablet_type, 'AsTransaction': as_transaction, } self._add_session(req) response = self._get_client().call('VTGate.ExecuteBatch', req) self._update_session(response) if response.reply.get('Error'): raise gorpc.AppError(response.reply['Error'], 'VTGate.ExecuteBatch') for reply in response.reply['List']: fields = [] conversions = [] results = [] rowcount = 0 for field in reply['Fields']: fields.append((field['Name'], field['Type'])) conversions.append(field_types.conversions.get(field['Type'])) for row in reply['Rows']: results.append(tuple(_make_row(row, conversions))) rowcount = reply['RowsAffected'] lastrowid = reply['InsertId'] rowsets.append((results, rowcount, lastrowid, fields)) except gorpc.GoRpcError as e: self.logger_object.log_private_data(bind_variables_list) raise convert_exception(e, str(self), sql_list) except Exception: logging.exception('gorpc low-level error') raise return rowsets
def _execute(self, sql, bind_variables): new_binds = field_types.convert_bind_vars(bind_variables) req = { 'Sql': sql, 'BindVariables': new_binds, 'Keyspace': self.keyspace, 'TabletType': self.tablet_type, 'Shards': [self.shard], } self._add_session(req) fields = [] conversions = [] results = [] rowcount = 0 lastrowid = 0 try: response = self.client.call('VTGate.ExecuteShard', req) self._update_session(response) reply = response.reply # TODO(sougou): Simplify this check after all servers are deployed if 'Error' in response.reply and response.reply['Error']: raise gorpc.AppError(response.reply['Error'], 'VTGate.ExecuteShard') if 'Result' in reply: res = reply['Result'] for field in res['Fields']: fields.append((field['Name'], field['Type'])) conversions.append( field_types.conversions.get(field['Type'])) for row in res['Rows']: results.append(tuple(_make_row(row, conversions))) rowcount = res['RowsAffected'] lastrowid = res['InsertId'] except gorpc.GoRpcError as e: raise convert_exception(e, str(self), sql, bind_variables) except: logging.exception('gorpc low-level error') raise return results, rowcount, lastrowid, fields
def row_generator(): try: while True: try: stream_result = rpc_client.stream_next() if stream_result is None: break if stream_result.reply.get('Err'): drain_conn_after_streaming_app_error() raise gorpc.AppError( stream_result.reply['Err'].get( 'Message', 'Missing error message')) for result_item in stream_result.reply['Rows']: yield tuple( _make_row(result_item, stream_conversions)) except gorpc.GoRpcError as e: raise convert_exception(e, str(self)) except Exception: logging.exception('gorpc low-level error') raise finally: rpc_client.close()
def _stream_execute(self, sql, bind_variables): req = { 'Query': { 'Sql': sql, 'BindVariables': field_types.convert_bind_vars(bind_variables), 'SessionId': self.session_id, 'TransactionId': self.transaction_id }, 'ImmediateCallerID': { 'Username': self.caller_id } } self._stream_fields = [] self._stream_conversions = [] self._stream_result = None self._stream_result_index = 0 try: self.client.stream_call('SqlQuery.StreamExecute2', req) first_response = self.client.stream_next() reply = first_response.reply if reply.get('Err'): self.__drain_conn_after_streaming_app_error() raise gorpc.AppError(reply['Err'].get('Message', 'Missing error message')) for field in reply['Fields']: self._stream_fields.append((field['Name'], field['Type'])) self._stream_conversions.append( field_types.conversions.get(field['Type'])) except gorpc.GoRpcError as e: self.logger_object.log_private_data(bind_variables) raise convert_exception(e, str(self), sql) except: logging.exception('gorpc low-level error') raise return None, 0, 0, self._stream_fields
def _stream_execute(self, sql, bind_variables): req = { 'Query': { 'Sql': sql, 'BindVariables': field_types.convert_bind_vars(bind_variables), 'SessionId': self.session_id, 'TransactionId': self.transaction_id }, 'ImmediateCallerID': { 'Username': self.caller_id } } rpc_client = self._get_client() stream_fields = [] stream_conversions = [] def drain_conn_after_streaming_app_error(): """Drain connection of all incoming streaming packets (ignoring them). This is necessary for streaming calls which return application errors inside the RPC response (instead of through the usual GoRPC error return). This is because GoRPC always expects the last packet to be an error; either the usual GoRPC application error return, or a special "end-of-stream" error. If an application error is returned with the RPC response, there will still be at least one more packet coming, as GoRPC has not seen anything that it considers to be an error. If the connection is not drained of this last packet, future reads from the wire will be off by one and will return errors. """ next_result = rpc_client.stream_next() if next_result is not None: rpc_client.close() raise gorpc.GoRpcError( 'Connection should only have one packet remaining' ' after streaming app error in RPC response.') try: rpc_client.stream_call('SqlQuery.StreamExecute2', req) first_response = rpc_client.stream_next() reply = first_response.reply if reply.get('Err'): drain_conn_after_streaming_app_error() raise gorpc.AppError(reply['Err'].get('Message', 'Missing error message')) for field in reply['Fields']: stream_fields.append((field['Name'], field['Type'])) stream_conversions.append( field_types.conversions.get(field['Type'])) except gorpc.GoRpcError as e: self.logger_object.log_private_data(bind_variables) raise convert_exception(e, str(self), sql) except Exception: logging.exception('gorpc low-level error') raise # Take the BsonRpcClient from VTGateConnection. The row_generator # will manage the BsonRpcClient. This VTGateConnection will connect # to a new client if needed. self.client = None def row_generator(): try: while True: try: stream_result = rpc_client.stream_next() if stream_result is None: break if stream_result.reply.get('Err'): drain_conn_after_streaming_app_error() raise gorpc.AppError( stream_result.reply['Err'].get( 'Message', 'Missing error message')) for result_item in stream_result.reply['Rows']: yield tuple( _make_row(result_item, stream_conversions)) except gorpc.GoRpcError as e: raise convert_exception(e, str(self)) except Exception: logging.exception('gorpc low-level error') raise finally: rpc_client.close() return row_generator(), stream_fields