def make_client(service,
host="localhost",
port=9090,
unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=None,
cafile=None,
ssl_context=None,
certfile=None,
keyfile=None):
if unix_socket:
socket = TSocket(unix_socket=unix_socket)
if certfile:
warnings.warn("SSL only works with host:port, not unix_socket.")
elif host and port:
if cafile or ssl_context:
socket = TSSLSocket(host,
port,
socket_timeout=timeout,
cafile=cafile,
certfile=certfile,
keyfile=keyfile,
ssl_context=ssl_context)
else:
socket = TSocket(host, port, socket_timeout=timeout)
else:
raise ValueError("Either host/port or unix_socket must be provided.")
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
def server(request):
p1 = TProcessor(mux.ThingOneService, DispatcherOne())
p2 = TProcessor(mux.ThingTwoService, DispatcherTwo())
mux_proc = TMultiplexedProcessor()
mux_proc.register_processor("ThingOneService", p1)
mux_proc.register_processor("ThingTwoService", p2)
_server = TThreadedServer(mux_proc,
TServerSocket(unix_socket=sock_path),
iprot_factory=TBinaryProtocolFactory(),
itrans_factory=TBufferedTransportFactory())
ps = multiprocessing.Process(target=_server.serve)
ps.start()
time.sleep(0.1)
def fin():
if ps.is_alive():
ps.terminate()
try:
os.remove(sock_path)
except IOError:
pass
request.addfinalizer(fin)
def make_server(service,
handler,
host="localhost",
port=9090,
unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
client_timeout=3000,
certfile=None):
processor = TProcessor(service, handler)
if unix_socket:
server_socket = TServerSocket(unix_socket=unix_socket)
if certfile:
warnings.warn("SSL only works with host:port, not unix_socket.")
elif host and port:
if certfile:
server_socket = TSSLServerSocket(host=host,
port=port,
client_timeout=client_timeout,
certfile=certfile)
else:
server_socket = TServerSocket(host=host,
port=port,
client_timeout=client_timeout)
else:
raise ValueError("Either host/port or unix_socket must be provided.")
server = TThreadedServer(processor,
server_socket,
iprot_factory=proto_factory,
itrans_factory=trans_factory)
return server
def get_nimbus_client(env_config=None, host=None, port=None, timeout=7000):
"""Get a Thrift RPC client for Nimbus given project's config file.
:param env_config: The project's parsed config.
:type env_config: `dict`
:param host: The host to use for Nimbus. If specified, `env_config` will
not be consulted.
:type host: `str`
:param port: The port to use for Nimbus. If specified, `env_config` will
not be consulted.
:type port: `int`
:param timeout: The time to wait (in milliseconds) for a response from
Nimbus.
:param timeout: `int`
:returns: a ThriftPy RPC client to use to communicate with Nimbus
"""
if host is None:
host, port = get_nimbus_host_port(env_config)
nimbus_client = make_client(
Nimbus,
host=host,
port=port,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TFramedTransportFactory(),
timeout=timeout,
)
return nimbus_client
def encode(n, proto_factory=TBinaryProtocolFactory()):
ab = make_addressbook()
start = time.time()
for i in range(n):
serialize(ab, proto_factory)
end = time.time()
print("encode\t-> {}".format(end - start))
def decode(n, proto_factory=TBinaryProtocolFactory()):
ab = addressbook.AddressBook()
start = time.time()
for i in range(n):
deserialize(ab, ab_encoded, proto_factory)
end = time.time()
print("decode\t-> {}".format(end - start))
def client_context(service,
host='localhost',
port=9090,
path='',
scheme='http',
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
ssl_context_factory=None,
http_header_factory=None,
timeout=DEFAULT_HTTP_CLIENT_TIMEOUT_MS,
url=''):
if url:
parsed_url = urllib.parse.urlparse(url)
host = parsed_url.hostname or host
port = parsed_url.port or port
scheme = parsed_url.scheme or scheme
path = parsed_url.path or path
if path and path[0] != "/":
# path should have `/` prefix, but we can make a compatible here.
path = "/" + path
uri = HTTP_URI.format(scheme=scheme, host=host, port=port, path=path)
http_socket = THttpClient(uri, timeout, ssl_context_factory,
http_header_factory)
transport = trans_factory.get_transport(http_socket)
try:
iprot = proto_factory.get_protocol(transport)
transport.open()
yield TClient(service, iprot)
finally:
transport.close()
def client_two(timeout=3000):
binary_factory = TBinaryProtocolFactory()
multiplexing_factory = TMultiplexedProtocolFactory(binary_factory,
"ThingTwoService")
return client_context(mux.ThingTwoService,
unix_socket=sock_path,
timeout=timeout,
proto_factory=multiplexing_factory)
def make_server(service, handler, host, port,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory()):
processor = TProcessor(service, handler)
server = THttpServer(processor, (host, port),
itrans_factory=trans_factory,
iprot_factory=proto_factory)
return server
def client_context(service,
host="localhost",
port=9090,
unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=None,
socket_timeout=3000,
connect_timeout=3000,
cafile=None,
ssl_context=None,
certfile=None,
keyfile=None,
url=""):
if url:
parsed_url = urllib.parse.urlparse(url)
host = parsed_url.hostname or host
port = parsed_url.port or port
if timeout:
warnings.warn("`timeout` deprecated, use `socket_timeout` and "
"`connect_timeout` instead.")
socket_timeout = connect_timeout = timeout
if unix_socket:
socket = TSocket(unix_socket=unix_socket,
connect_timeout=connect_timeout,
socket_timeout=socket_timeout)
if certfile:
warnings.warn("SSL only works with host:port, not unix_socket.")
elif host and port:
if cafile or ssl_context:
socket = TSSLSocket(host,
port,
connect_timeout=connect_timeout,
socket_timeout=socket_timeout,
cafile=cafile,
certfile=certfile,
keyfile=keyfile,
ssl_context=ssl_context)
else:
socket = TSocket(host,
port,
connect_timeout=connect_timeout,
socket_timeout=socket_timeout)
else:
raise ValueError(
"Either host/port or unix_socket or url must be provided.")
try:
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
yield TClient(service, protocol)
finally:
transport.close()
def main() -> None:
Utils.configure_logger("log/server.log")
logger = logging.getLogger(__name__)
s = schema.Schema({
"--ipv6": bool,
"--port": schema.Use(int),
"--help": bool,
})
arguments = s.validate(docopt(__doc__))
current_script_dir = os.path.dirname(os.path.abspath(__file__))
thrift_path = os.path.join(current_script_dir,
"thrift/logging_service.thrift")
logger.info(f"Loading the thrift definition from: {thrift_path};")
logging_service_thrift = thriftpy2.load(
thrift_path,
module_name="logging_service_thrift",
)
if arguments["--ipv6"]:
socket_family = socket.AF_INET6
any_host_interface = "::"
socket_family_name = "IPv6"
else:
socket_family = socket.AF_INET
any_host_interface = "0.0.0.0"
socket_family_name = "IPv4"
server_port = arguments["--port"]
queue_manager = MemoryQueueManager()
logging_client = MetaLoggingClient()
metadata_manager = MetadataManager(queue_manager, logging_client)
handler = LoggingServiceHandler(metadata_manager, logging_client,
logging_service_thrift)
proc = TProcessor(logging_service_thrift.LoggingService, handler)
server = TThreadedServer(
proc,
TServerSocket(
host=any_host_interface,
socket_family=socket_family,
port=server_port,
client_timeout=CLIENT_TIMEOUT_MS,
),
iprot_factory=TBinaryProtocolFactory(),
itrans_factory=TBufferedTransportFactory(),
)
logger.info(
f"Logging service server listens host={any_host_interface}[{socket_family_name}], port={server_port}."
)
server.serve()
logger.info("done.")
def __init__(self,
address: str,
send_interval: float = 5,
*,
send_max_size: int = 100,
send_attempt_count: int = 3,
send_timeout: Optional[aiohttp.ClientTimeout] = None) -> None:
if send_timeout is None:
send_timeout = DEFAULT_TIMEOUT
self._address = address
self._batch_manager = BatchManager(send_max_size, send_interval,
send_attempt_count, self._send_data)
self._session = aiohttp.ClientSession(
timeout=send_timeout,
headers={hdrs.CONTENT_TYPE: "application/x-thrift"},
)
self._binary = TBinaryProtocolFactory(strict_read=False)
def main():
dd_proc = TProcessor(dd_thrift.DingService, DingDispatcher())
pp_proc = TProcessor(pp_thrift.PingService, PingDispatcher())
mux_proc = TMultiplexedProcessor()
mux_proc.register_processor(DD_SERVICE_NAME, dd_proc)
mux_proc.register_processor(PP_SERVICE_NAME, pp_proc)
server = TThreadedServer(mux_proc, TServerSocket(),
iprot_factory=TBinaryProtocolFactory(),
itrans_factory=TBufferedTransportFactory())
server.serve()
def gen_server(port, tracker=tracker, processor=TTrackedProcessor):
args = [processor, addressbook.AddressBookService, Dispatcher()]
if tracker:
args.insert(1, tracker)
processor = TProcessorFactory(*args)
server_socket = TServerSocket(host="localhost", port=port)
server = TSampleServer(processor, server_socket,
prot_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory())
ps = multiprocessing.Process(target=server.serve)
ps.start()
return ps, server
def client(client_class=TTrackedClient, port=PORT):
socket = TSocket("localhost", port)
try:
trans = TBufferedTransportFactory().get_transport(socket)
proto = TBinaryProtocolFactory().get_protocol(trans)
trans.open()
args = [addressbook.AddressBookService, proto]
if client_class.__name__ == TTrackedClient.__name__:
args.insert(0, SampleTracker("test_client", "test_server"))
yield client_class(*args)
finally:
trans.close()
def __init__(self,
processor,
trans,
itrans_factory=None,
iprot_factory=None,
otrans_factory=None,
oprot_factory=None):
self.processor = processor
self.trans = trans
self.itrans_factory = itrans_factory or TBufferedTransportFactory()
self.iprot_factory = iprot_factory or TBinaryProtocolFactory()
self.otrans_factory = otrans_factory or self.itrans_factory
self.oprot_factory = oprot_factory or self.iprot_factory
def make_client(
service,
host,
port,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TFramedTransportFactory(),
socket_factory=TNonBlockingSocket,
socket_timeout=1000,
):
socket = socket_factory(host, port, socket_timeout=socket_timeout)
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
def make_client(service, host='localhost', port=9090, path='', scheme='http',
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
ssl_context_factory=None,
timeout=DEFAULT_HTTP_CLIENT_TIMEOUT_MS, url=''):
if url:
parsed_url = urllib.parse.urlparse(url)
host = parsed_url.hostname or host
port = parsed_url.port or port
scheme = parsed_url.scheme or scheme
path = parsed_url.path or path
uri = HTTP_URI.format(scheme=scheme, host=host, port=port, path=path)
http_socket = THttpClient(uri, timeout, ssl_context_factory)
transport = trans_factory.get_transport(http_socket)
iprot = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, iprot)
class Connection:
# TODO: Auto generate these?
THRIFT_TRANSPORTS = dict(
buffered=TBufferedTransportFactory(),
framed=TFramedTransportFactory(),
)
THRIFT_PROTOCOLS = dict(
binary=TBinaryProtocolFactory(decode_response=False),
compact=TCompactProtocolFactory(decode_response=False),
)
THRIFT_CLIENTS = dict(
socket=make_client,
http=make_http_client,
)
def _autoconnect(self):
self.open()
def make_client(service,
host="localhost",
port=9090,
unix_socket=None,
proto_factory=TBinaryProtocolFactory(),
trans_factory=TBufferedTransportFactory(),
timeout=3000,
cafile=None,
ssl_context=None,
certfile=None,
keyfile=None,
url="",
socket_family=socket.AF_INET):
if url:
parsed_url = urllib.parse.urlparse(url)
host = parsed_url.hostname or host
port = parsed_url.port or port
if unix_socket:
socket = TSocket(unix_socket=unix_socket, socket_timeout=timeout)
if certfile:
warnings.warn("SSL only works with host:port, not unix_socket.")
elif host and port:
if cafile or ssl_context:
socket = TSSLSocket(host,
port,
socket_timeout=timeout,
socket_family=socket_family,
cafile=cafile,
certfile=certfile,
keyfile=keyfile,
ssl_context=ssl_context)
else:
socket = TSocket(host,
port,
socket_family=socket_family,
socket_timeout=timeout)
else:
raise ValueError(
"Either host/port or unix_socket or url must be provided.")
transport = trans_factory.get_transport(socket)
protocol = proto_factory.get_protocol(transport)
transport.open()
return TClient(service, protocol)
def main():
binary_factory = TBinaryProtocolFactory()
dd_factory = TMultiplexedProtocolFactory(binary_factory, DD_SERVICE_NAME)
with client_context(dd_thrift.DingService,
'127.0.0.1',
9090,
proto_factory=dd_factory) as c:
# ring that doorbell
dong = c.ding()
print(dong)
pp_factory = TMultiplexedProtocolFactory(binary_factory, PP_SERVICE_NAME)
with client_context(pp_thrift.PingService,
'127.0.0.1',
9090,
proto_factory=pp_factory) as c:
# play table tennis like a champ
pong = c.ping()
print(pong)
class ThriftTransport(TransportABC):
MODULE_PATH = os.path.abspath(os.path.dirname(aiojaeger.__file__))
jaeger_thrift = thriftpy2.load(
os.path.join(MODULE_PATH, "jaeger-idl/thrift/jaeger.thrift"),
module_name="jaeger_thrift",
)
def __init__(
self,
address: str,
send_interval: float = 5,
*,
send_max_size: int = 100,
send_attempt_count: int = 3,
send_timeout: Optional[aiohttp.ClientTimeout] = None
) -> None:
if send_timeout is None:
send_timeout = DEFAULT_TIMEOUT
self._address = address
self._batch_manager = BatchManager(
send_max_size, send_interval, send_attempt_count, self._send_data
)
self._session = aiohttp.ClientSession(
timeout=send_timeout,
headers={hdrs.CONTENT_TYPE: "application/x-thrift"},
)
self._binary = TBinaryProtocolFactory(strict_read=False)
async def close(self) -> None:
self._closing = True
await self._batch_manager.stop()
def send(self, record: Record) -> None:
self._batch_manager.add(record)
async def _send_data(self, data: List[Record]) -> bool:
if not data:
return True
batch = self.jaeger_thrift.Batch()
process = self.jaeger_thrift.Process()
process.serviceName = data[0].service_name
batch.process = process
spans = []
for record in data:
span = record.asthrift(self.jaeger_thrift)
spans.append(span)
batch.spans = spans
otrans = TMemoryBuffer()
self._binary.get_protocol(otrans).write_struct(batch)
resp = await self._session.post(self._address, data=otrans.getvalue())
try:
resp.raise_for_status()
except ClientResponseError:
text = await resp.text()
logger.exception("Can not send spans to jaeger: %r", text)
return False
return True
def generate_trace_id(self) -> int:
# TODO: support 128 bit header
return random_id()
def generate_span_id(self) -> int:
return random_id()
@property
def span_context(self) -> Type[BaseTraceContext]:
return JaegerTraceContext
def get_protoco_factory(self):
from thriftpy2.protocol import TBinaryProtocolFactory
return TBinaryProtocolFactory().get_protocol
def serialize_bin(thrift_obj: object) -> bytes:
return bytes(serialize(
thrift_obj,
proto_factory=TBinaryProtocolFactory(),
))
