def handle(self, listener, client, addr):
if self.app.cfg.thrift_client_timeout is not None:
client.settimeout(self.app.cfg.thrift_client_timeout)
result = TSocket()
result.set_handle(client)
try:
itrans = self.app.tfactory.get_transport(result)
otrans = self.app.tfactory.get_transport(result)
iprot = self.app.pfactory.get_protocol(itrans)
oprot = self.app.pfactory.get_protocol(otrans)
processor = self.get_thrift_processor()
try:
while True:
processor.process(iprot, oprot)
self.notify()
except TTransportException:
pass
except socket.error as e:
if e.args[0] == errno.ECONNRESET:
self.log.debug(e)
else:
self.log.exception(e)
except Exception as e:
self.log.exception(e)
finally:
itrans.close()
otrans.close()
def test_read_long_data():
val = 'z' * 97 * 1024
def serve():
server_sock = TServerSocket(
unix_socket="./thriftpy_test.sock")
server_sock.listen()
client = server_sock.accept()
t = TCyBufferedTransport(client)
proto.write_val(t, TType.STRING, val)
t.flush()
p = multiprocessing.Process(target=serve)
p.start()
time.sleep(0.1)
try:
sock = TSocket(unix_socket="./thriftpy_test.sock")
b = TCyBufferedTransport(sock)
b.open()
assert val == proto.read_val(b, TType.STRING)
sock.close()
finally:
p.terminate()
try:
os.remove("./thriftpy_test.sock")
except IOError:
pass
def handle(self, listener, client, addr):
self.cfg.on_connected(self, addr)
if self.app.cfg.thrift_client_timeout is not None:
client.settimeout(self.app.cfg.thrift_client_timeout)
result = TSocket()
result.set_handle(client)
try:
itrans = self.app.tfactory.get_transport(result)
otrans = self.app.tfactory.get_transport(result)
iprot = self.app.pfactory.get_protocol(itrans)
oprot = self.app.pfactory.get_protocol(otrans)
processor = self.get_thrift_processor()
try:
while True:
processor.process(iprot, oprot)
self.notify()
except TTransportException:
pass
except socket.error as e:
if e.args[0] == errno.ECONNRESET:
self.log.debug(e)
else:
self.log.exception(e)
except Exception as e:
self.log.exception(e)
finally:
itrans.close()
otrans.close()
def test_read_long_data():
val = 'z' * 97 * 1024
unix_sock = "/tmp/thriftpy_test.sock"
def serve():
server_sock = TServerSocket(unix_socket=unix_sock)
server_sock.listen()
client = server_sock.accept()
t = TCyBufferedTransport(client)
proto.write_val(t, TType.STRING, val)
t.flush()
# wait for client to read
time.sleep(1)
p = multiprocessing.Process(target=serve)
p.start()
time.sleep(0.1)
try:
sock = TSocket(unix_socket=unix_sock)
b = TCyBufferedTransport(sock)
b.open()
assert val == proto.read_val(b, TType.STRING)
sock.close()
finally:
p.terminate()
try:
os.remove(unix_sock)
except IOError:
pass
def client_context(service, host="localhost", port=9090, unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=None, socket_timeout=3000, connect_timeout=3000,
cafile=None, ssl_context=None, certfile=None, keyfile=None):
if timeout:
warnings.warn("`timeout` deprecated, use `socket_timeout` and "
"`connect_timeout` instead.")
socket_timeout = connect_timeout = timeout
if unix_socket:
socket = TSocket(unix_socket=unix_socket,
connect_timeout=connect_timeout,
socket_timeout=socket_timeout)
elif host and port:
socket = TSocket(host, port,
connect_timeout=connect_timeout,
socket_timeout=socket_timeout)
else:
raise ValueError("Either host/port or unix_socket must be provided.")
try:
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
yield TClient(service, protocol)
finally:
transport.close()
def make_client(service,
host="localhost",
port=9090,
unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=None,
cafile=None,
ssl_context=None,
certfile=None,
keyfile=None):
if unix_socket:
socket = TSocket(unix_socket=unix_socket)
if certfile:
warnings.warn("SSL only works with host:port, not unix_socket.")
elif host and port:
if cafile or ssl_context:
socket = TSSLSocket(host,
port,
socket_timeout=timeout,
cafile=cafile,
certfile=certfile,
keyfile=keyfile,
ssl_context=ssl_context)
else:
socket = TSocket(host, port, socket_timeout=timeout)
else:
raise ValueError("Either host/port or unix_socket must be provided.")
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
def __getattr__(self, item):
socket = TSocket(self.host, self.port)
socket.set_timeout(self.timeout)
transport = TRANS_FACTORY().get_transport(socket)
protocol = PROTO_FACTORY().get_protocol(transport)
client = TClient(self.service, protocol)
attr = getattr(client, item)
return _wrapper_api(attr, transport)
def _refresh_thrift_client(self):
"""Refresh the Thrift socket, transport, and client."""
socket = TSocket(self.host, self.port)
if self.timeout is not None:
socket.set_timeout(self.timeout)
self.transport = self._transport_class(socket)
protocol = self._protocol_class(self.transport, decode_response=False)
self.client = TClient(Hbase, protocol)
def _refresh_thrift_client(self):
"""Refresh the Thrift socket, transport, and client."""
socket = TSocket(self.host, self.port)
if self.timeout is not None:
socket.set_timeout(self.timeout)
self.transport = self._transport_class(socket)
protocol = self._protocol_class(self.transport, decode_response=False)
self.client = TClient(Hbase, protocol)
def make_client(service, host, port,
proto_factory=TBinaryProtocolFactory(),
transport_factory=TBufferedTransportFactory(),
timeout=None):
socket = TSocket(host, port)
if timeout:
socket.set_timeout(timeout)
transport = transport_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
def make_client(service, host, port,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=None):
socket = TSocket(host, port)
if timeout:
socket.set_timeout(timeout)
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
def client_context(service, host, port,
proto_factory=TBinaryProtocolFactory(),
transport_factory=TBufferedTransportFactory(),
timeout=None):
try:
socket = TSocket(host, port)
if timeout:
socket.set_timeout(timeout)
transport = transport_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
yield TClient(service, protocol)
finally:
transport.close()
def client_context(service, host, port,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=None):
try:
socket = TSocket(host, port)
if timeout:
socket.set_timeout(timeout)
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
yield TClient(service, protocol)
finally:
transport.close()
def make_client(service, host="localhost", port=9090, unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=None):
if unix_socket:
socket = TSocket(unix_socket=unix_socket)
elif host and port:
socket = TSocket(host, port, socket_timeout=timeout)
else:
raise ValueError("Either host/port or unix_socket must be provided.")
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
def get_new_client(self):
"""
轮询在每个ip:port的连接池中获取连接（线程安全）
从当前队列右侧取出ip:port信息，获取client
将连接池对象放回到当前队列的左侧
请求或连接超时时间，默认30秒
:return:
"""
with self.lock:
if self.pool_size < self.maxActive:
try:
ip = self.load_balance_queue.pop()
except IndexError:
raise CTECThriftClientError('没有可用的服务提供者列表！')
if ip:
self.load_balance_queue.appendleft(ip)
# 创建新的thrift client
t_socket = TSocket(ip.split(':')[0], int(ip.split(':')[1]),
socket_timeout=1000 * self.socket_timeout)
proto_factory = TBinaryProtocolFactory()
trans_factory = TBufferedTransportFactory()
transport = trans_factory.get_transport(t_socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
client = TClient(self.service, protocol)
self.pool_size += 1
return client
else:
return None
def test_transport_handle():
from thriftpy._compat import CYTHON
if not CYTHON:
return
from thriftpy.transport import TSocket
from thriftpy.transport.memory import TCyMemoryBuffer
s = TSocket()
s.set_handle('the sock')
assert TCyBufferedTransport(s).sock == 'the sock'
assert TCyFramedTransport(s).sock == 'the sock'
assert TCyMemoryBuffer().sock is None
assert TCyBufferedTransport(TCyFramedTransport(s)).sock == 'the sock'
assert TCyBufferedTransport(TCyMemoryBuffer()).sock is None
def __create_client(self):
socket = TSocket(self.__host,
self.__port,
socket_timeout=self.__timeout)
self.__transport = TCyFramedTransportFactory().get_transport(socket)
protocol = TCyBinaryProtocolFactory().get_protocol(self.__transport)
self.__client = TClient(recs_thrift.RecommendationsService, protocol)
def client_context(service, host, port,
proto_factory=TBinaryProtocolFactory()):
try:
transport = TBufferedTransport(TSocket(host, port))
protocol = proto_factory.get_protocol(transport)
transport.open()
yield TClient(service, protocol)
finally:
transport.close()
def make_client(service, host="localhost", port=9090, unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=None):
if unix_socket:
socket = TSocket(unix_socket=unix_socket)
elif host and port:
socket = TSocket(host, port)
else:
raise ValueError("Either host/port or unix_socket must be provided.")
if timeout:
socket.set_timeout(timeout)
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
def test_exceptions(monkeypatch):
import socket
from folklore.service import FolkloreService, Processor, \
CloseConnectionError, FolkloreBinaryProtocol
from thriftpy.transport import TTransportException, TSocket
from thriftpy.protocol.exc import TProtocolException
service = FolkloreService()
service.api_map = {}
service.context.update({
'client_addr': 'localhost',
'client_port': 1,
})
t_exc = TTransportException()
t_exc.message = 'end of file'
sock = TSocket()
sock.sock = socket.socket()
mock_close = mock.Mock()
monkeypatch.setattr(FolkloreBinaryProtocol, 'close', mock_close)
t_exc.type = TTransportException.END_OF_FILE
with mock.patch.object(Processor, 'process', side_effect=t_exc):
service.run(sock)
service.logger = mock.Mock(exception=mock.Mock())
t_exc.type = TTransportException.ALREADY_OPEN
with mock.patch.object(Processor, 'process', side_effect=t_exc):
service.run(sock)
service.logger.exception.assert_called_with(t_exc)
p_exc = TProtocolException()
p_exc.type = TProtocolException.BAD_VERSION
with mock.patch.object(Processor, 'process', side_effect=p_exc):
service.run(sock)
service.logger.warn.assert_called_with('[%s:%s] protocol error: %s',
'localhost', 1, p_exc)
c_exc = CloseConnectionError()
with mock.patch.object(Processor, 'process', side_effect=c_exc):
service.run(sock)
mock_close.assert_called_with()
def __init__(self):
super(ThriftLocust, self).__init__()
socket = TSocket('127.0.0.1', 9999)
proto_factory = TBinaryProtocolFactory()
trans_factory = TBufferedTransportFactory()
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
hash_thrift = thriftpy.load("hash.thrift", module_name="hash_thrift")
self.client = ThriftClient(hash_thrift.HashService, protocol)
def handle(self, listener, client, addr):
self.cfg.on_connected(self, addr)
if self.app.cfg.thrift_client_timeout is not None:
client.settimeout(self.app.cfg.thrift_client_timeout)
result = TSocket()
result.set_handle(client)
try:
itrans = self.app.tfactory.get_transport(result)
otrans = self.app.tfactory.get_transport(result)
iprot = self.app.pfactory.get_protocol(itrans)
oprot = self.app.pfactory.get_protocol(otrans)
processor = self.get_thrift_processor()
try:
while True:
processor.process(iprot, oprot)
except TTransportException:
pass
except (TProtocolException, ProtocolError) as err:
self.log.warning("Protocol error, is client(%s) correct? %s", addr,
err)
except TDecodeException as err:
self.log.exception('%r: %r', addr, err)
self.cfg.on_tdecode_exception(err)
except socket.timeout:
self.log.warning('Client timeout: %r', addr)
except socket.error as e:
if e.args[0] == errno.ECONNRESET:
self.log.debug('%r: %r', addr, e)
elif e.args[0] == errno.EPIPE:
self.log.warning('%r: %r', addr, e)
else:
self.log.exception('%r: %r', addr, e)
except Exception as e:
self.log.exception('%r: %r', addr, e)
finally:
itrans.close()
otrans.close()
self.cfg.post_connect_closed(self)
def open(self):
if self.transport and self.transport.is_open():
logger.debug("transport was opened.")
return
logger.debug("new transport...")
socket = TSocket(host=self.host, port=self.port, socket_timeout=self.timeout)
self.transport = self.transport_class(socket)
self.protocol = self.protocol_class(self.transport)
self.client = TClient(hbase_thrift.Hbase, self.protocol)
self.transport.open()
def __init__(self, service, socket_config, service_name=None):
trans_socket = TSocket(**socket_config)
self.__transport = TBufferedTransportFactory()\
.get_transport(trans_socket)
if service_name:
protocol_factory = TMultiplexedProtocolFactory(
TBinaryProtocolFactory(), service_name)
else:
protocol_factory = TBinaryProtocolFactory()
protocol = protocol_factory.get_protocol(self.__transport)
self.__client = TClient(service, protocol)
self.__is_open = False
def client_context(service,
host="localhost",
port=9090,
unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=None):
if unix_socket:
socket = TSocket(unix_socket=unix_socket)
elif host and port:
socket = TSocket(host, port)
else:
raise ValueError("Either host/port or unix_socket must be provided.")
if timeout:
socket.set_timeout(timeout)
try:
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
yield TClient(service, protocol)
finally:
transport.close()
def client(client_class=TTrackedClient, port=PORT):
socket = TSocket("localhost", port)
try:
trans = TBufferedTransportFactory().get_transport(socket)
proto = TBinaryProtocolFactory().get_protocol(trans)
trans.open()
args = [addressbook.AddressBookService, proto]
if client_class.__name__ == TTrackedClient.__name__:
args.insert(0, tracker)
yield client_class(*args)
finally:
trans.close()
def handle(self, listener, client, addr):
self.cfg.on_connected(self, addr)
if self.app.cfg.thrift_client_timeout is not None:
client.settimeout(self.app.cfg.thrift_client_timeout)
result = TSocket()
result.set_handle(client)
try:
itrans = self.app.tfactory.get_transport(result)
otrans = self.app.tfactory.get_transport(result)
iprot = self.app.pfactory.get_protocol(itrans)
oprot = self.app.pfactory.get_protocol(otrans)
processor = self.get_thrift_processor()
try:
while True:
processor.process(iprot, oprot)
except TTransportException:
pass
except (TProtocolException, ProtocolError) as err:
self.log.warning(
"Protocol error, is client(%s) correct? %s", addr, err
)
except socket.error as e:
if e.args[0] == errno.ECONNRESET:
self.log.debug('%r: %r', addr, e)
elif e.args[0] == errno.EPIPE:
self.log.warning('%r: %r', addr, e)
else:
self.log.exception('%r: %r', addr, e)
except Exception as e:
self.log.exception('%r: %r', addr, e)
finally:
itrans.close()
otrans.close()
self.cfg.post_connect_closed(self)
def handle(self, listener, client, addr):
if self.app.cfg.thrift_client_timeout is not None:
client.settimeout(self.app.cfg.thrift_client_timeout)
result = TSocket()
result.set_handle(client)
try:
itrans = self.app.tfactory.get_transport(result)
otrans = self.app.tfactory.get_transport(result)
iprot = self.app.pfactory.get_protocol(itrans)
oprot = self.app.pfactory.get_protocol(otrans)
try:
while True:
self.app.thrift_app.process(iprot, oprot)
except TTransportException:
pass
except Exception as e:
logger.exception(e)
finally:
itrans.close()
otrans.close()
def init_client(self, ip, port):
"""Establish a connection to thrift server of ZLP Service. Init client opening sockets and init events handler.
Args:
ip (str): ipv6 network address of ZLP-Service
port (str): port number on which ZLP-Service listens for requests
"""
client_socket = TSocket(ip,
port,
socket_family=socket.AF_INET,
socket_timeout=50000)
transport = TBufferedTransportFactory().get_transport(client_socket)
protocol = TBinaryProtocolFactory().get_protocol(transport)
transport.open()
super().__init__(self.thrift_interface.ServiceInterface, protocol)
def handle(self, listener, client, addr):
self.cfg.on_connected(self, addr)
if self.app.cfg.thrift_client_timeout is not None:
client.settimeout(self.app.cfg.thrift_client_timeout)
result = TSocket()
result.set_handle(client)
try:
itrans = self.app.transport_factory.get_transport(result)
otrans = self.app.transport_factory.get_transport(result)
iprot = self.app.protocol_factory.get_protocol(itrans)
oprot = self.app.protocol_factory.get_protocol(otrans)
processor = self.get_thrift_processor()
try:
while True:
self.handle_processor_receive(listener, client, addr)
processor.process(iprot, oprot)
self.notify()
except TTransportException:
pass
# except thrift.Thrift.TException as e:
# self.log.error(e.message)
except socket.error as e:
if e.args[0] == errno.ECONNRESET:
self.log.debug(e)
else:
self.log.exception(e)
except Exception as e:
self.log.exception(e)
finally:
itrans.close()
otrans.close()
self.cfg.post_connect_closed(self)
def _refresh_thrift_client(self):
"""Refresh the Thrift socket, transport, and client."""
socket = TSocket(host=self.host,
port=self.port,
socket_timeout=self.timeout)
self.transport = self._transport_class(socket)
if self.use_kerberos:
self.transport = TSaslClientTransport(self.transport, self.host,
self.sasl_service_name)
sasl_auth = 'GSSAPI'
def sasl_factory():
sasl_client = sasl.Client()
sasl_client.setAttr('host', self.host)
sasl_client.setAttr('service', self.sasl_service_name)
sasl_client.init()
return sasl_client
self.transport = TSaslClientTransport(sasl_factory, sasl_auth,
socket)
protocol = self._protocol_class(self.transport, decode_response=False)
self.client = TClient(Hbase, protocol)
def make_client(service, host, port, proto_factory=TBinaryProtocolFactory()):
transport = TBufferedTransport(TSocket(host, port))
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
