def test_writer_multiplexing(): server = TChannel('server') server.listen() received = {'chunked': False, 'singleframe': False} @server.raw.register('chunked') def chunked(request): received['chunked'] = True return b'chunked' @server.raw.register('singleframe') def singleframe(request): received['singleframe'] = True return b'singleframe' client = TChannel('client') chunked_future = client.raw( 'server', 'chunked', bytes([0x00] * 1024 * 1024), # 1 MB = 16 frames hostport=server.hostport, timeout=0.5, ) yield client.raw( 'server', 'singleframe', b'\x00', # single frame hostport=server.hostport, ) assert received['singleframe'] assert not received['chunked'] yield chunked_future assert received['chunked']
def test_connection_close(mock_server): tchannel = TChannel(name="test") # use a bad call to finish the hand shake and build the connection. with pytest.raises(BadRequestError): yield tchannel.raw(service="test-service", hostport=mock_server.hostport, endpoint="testg") # close the server and close the connection. mock_server.tchannel._dep_tchannel.close() with pytest.raises(NetworkError): yield tchannel.raw(service="test-service", hostport=mock_server.hostport, endpoint="testg")
def test_timeout_during_handshake_is_retried(timeout_server): tchannel = TChannel(name='client', known_peers=[timeout_server]) # We want the client to look for other peers if an INIT times out rather # than raising a timeout error so we expect a NoAvailablePeerError here. with patch.object(connection, 'DEFAULT_INIT_TIMEOUT_SECS', 0.1): with pytest.raises(errors.NoAvailablePeerError): yield tchannel.raw(service='server', endpoint='endpoint')
def test_client_connection_change_callback(): server = TChannel('server') server.listen() @server.raw.register def hello(request): return 'hi' client = TChannel('client') count = [0] def test_cb(peer): count[0] += 1 client._dep_tchannel.peers.get( server.hostport)._on_conn_change_cb = test_cb yield client.raw( hostport=server.hostport, body='work', endpoint='hello', service='server' ) # 1: connection built, 1: sending request, 1: finish sending request assert count[0] == 3
def test_timeout_should_raise_timeout_error(): # Given this test server: server = TChannel(name='server') @server.register(scheme=schemes.RAW) @gen.coroutine def endpoint(request): yield gen.sleep(0.05) raise gen.Return('hello') server.listen() # Make a call: tchannel = TChannel(name='client') # timeout is less than server, should timeout with pytest.raises(TimeoutError): yield tchannel.call( scheme=schemes.RAW, service='server', arg1='endpoint', hostport=server.hostport, timeout=0.02, ) # timeout is more than server, should not timeout yield tchannel.raw( service='server', endpoint='endpoint', hostport=server.hostport, timeout=0.1, )
def test_local_timeout_unconsumed_message(): """Verify that if the client has a local timeout and the server eventually sends the message, the client does not log an "Unconsumed message" warning. """ server = TChannel('server') @server.raw.register('hello') @gen.coroutine def hello(request): yield gen.sleep(0.07) raise gen.Return('eventual response') server.listen() client = TChannel('client') with pytest.raises(TimeoutError): yield client.raw( 'server', 'hello', 'world', timeout=0.05, hostport=server.hostport, # The server will take 70 milliseconds but we allow at most 50. ) # Wait for the server to send the late response and make sure it doesn't # log a warning. with mock.patch.object(connection.log, 'warn') as mock_warn: # :( yield gen.sleep(0.03) assert mock_warn.call_count == 0
def test_no_infinite_trace_submit(): """Zipkin submissions must not trace themselves.""" def submit(request): return TResponse(True) zipkin_server = TChannel('zipkin') zipkin_server.thrift.register(TCollector, handler=submit) zipkin_server.listen() class TestTraceHook(EventHook): def __init__(self): self.tracings = [] def before_send_request(self, request): # if request.service == 'tcollector': self.tracings.append(request.tracing) server = TChannel('server', known_peers=[zipkin_server.hostport]) server.hooks.register(ZipkinTraceHook(tchannel=server, sample_rate=1)) test_trace_hook = TestTraceHook() server.hooks.register(test_trace_hook) server.thrift.register(TCollector, handler=submit) @server.raw.register @tornado.gen.coroutine def hello(request): if request.body == 'body': yield server.raw( service='server', endpoint='hello', body='boy', hostport=server.hostport, trace=True, ) raise tornado.gen.Return('hello') server.listen() client = TChannel('client') client.thrift.register(TCollector, handler=submit) yield client.raw( service='server', endpoint='hello', hostport=server.hostport, body='body', trace=True, ) # Continue yielding to the IO loop to allow our zipkin traces to be # handled. for _ in xrange(100): yield tornado.gen.moment # One trace for 'hello' and then 3 submissions to tcollector (1 time as # client, 2 times as server) assert len(test_trace_hook.tracings) == 4, test_trace_hook.tracings
def test_endpoint_not_found(mock_server): tchannel = TChannel(name='test') with pytest.raises(BadRequestError): yield tchannel.raw( service='test-server', endpoint='fooo', hostport=mock_server.hostport, )
def test_pending_outgoing(): server = TChannel('server') server.listen() @server.raw.register def hello(request): assert server._dep_tchannel.peers.peers[0].total_outbound_pendings == 1 return 'hi' client = TChannel('client') yield client.raw( hostport=server.hostport, body='work', endpoint='hello', service='server' ) client_peer = client._dep_tchannel.peers.peers[0] server_peer = server._dep_tchannel.peers.peers[0] assert client_peer.total_outbound_pendings == 0 assert server_peer.total_outbound_pendings == 0 class FakeMessageFactory(MessageFactory): def build_raw_message(self, context, args, is_completed=True): assert client_peer.total_outbound_pendings == 1 return super(FakeMessageFactory, self).build_raw_message( context, args, is_completed, ) client_conn = client_peer.connections[0] client_conn.request_message_factory = FakeMessageFactory( client_conn.remote_host, client_conn.remote_host_port, ) yield client.raw( hostport=server.hostport, body='work', endpoint='hello', service='server' ) assert client_peer.total_outbound_pendings == 0 assert server_peer.total_outbound_pendings == 0
def test_connection_close(mock_server): tchannel = TChannel(name='test') # use a bad call to finish the hand shake and build the connection. with pytest.raises(BadRequestError): yield tchannel.raw( service='test-service', hostport=mock_server.hostport, endpoint='testg', ) # close the server and close the connection. mock_server.tchannel._dep_tchannel.close() with pytest.raises(NetworkError): yield tchannel.raw( service='test-service', hostport=mock_server.hostport, endpoint='testg', )
def test_endpoint_not_found(): server = TChannel(name='server') server.listen() tchannel = TChannel(name='client') with pytest.raises(errors.BadRequestError): yield tchannel.raw( service='server', hostport=server.hostport, endpoint='foo', )
def test_per_request_caller_name_raw(): server = TChannel('server') server.listen() @server.raw.register('foo') def handler(request): assert request.transport.caller_name == 'bar' return b'success' client = TChannel('client', known_peers=[server.hostport]) res = yield client.raw('service', 'foo', b'', caller_name='bar') assert res.body == b'success'
def test_routing_delegate_is_propagated_raw(): server = TChannel('server') server.listen() @server.raw.register('foo') def handler(request): assert request.transport.routing_delegate == 'delegate' return b'success' client = TChannel('client', known_peers=[server.hostport]) res = yield client.raw('service', 'foo', b'', routing_delegate='delegate') assert res.body == b'success'
def test_tchannel_call_request_fragment(mock_server, arg2, arg3): endpoint = b"tchannelpeertest" mock_server.expect_call(endpoint).and_write(headers=endpoint, body=arg3) tchannel = TChannel(name="test") response = yield tchannel.raw( service="test-service", hostport=mock_server.hostport, endpoint=endpoint, headers=arg2, body=arg3 ) assert response.headers == endpoint assert response.body == arg3 assert response.transport.scheme == "raw"
def test_endpoint_not_found_with_raw_request(): server = TChannel(name='server') server.listen() tchannel = TChannel(name='client') with pytest.raises(errors.BadRequestError) as e: yield tchannel.raw( service='server', hostport=server.hostport, endpoint='foo', ) assert "Endpoint 'foo' is not defined" in e.value
def test_span_to_trace(tracer, mock_server): """ In this test we verify that if the tracer implementation supports the notions of trace_id, span_id, parent_id (similar to Zipkin) then we can pass those IDs not just via the headers (primary OpenTracing propagation mechanism), but also via TChannel's built-in tracing slot in the frame. :param tracer: injected BasicTracer mixin :param mock_server: injected TChannel mock server """ def span_to_trace(span): return { 'trace_id': span.context.trace_id, 'span_id': span.context.span_id, 'parent_span_id': span.context.parent_id, } context_provider = OpenTracingRequestContextProvider() hook = OpenTracingHook(tracer=tracer, context_provider=context_provider, span_to_trace_fn=span_to_trace) @mock_server.tchannel.raw.register('endpoint1') @tornado.gen.coroutine def handler1(request): ctx = mock_server.tchannel.context_provider.get_current_context() if hasattr(ctx, 'parent_tracing'): res = ctx.parent_tracing.trace_id else: res = 'unknown' raise tornado.gen.Return(Response('%s' % res)) tchannel = TChannel(name='test', context_provider=context_provider) tchannel.hooks.register(hook) with mock.patch('opentracing.tracer', tracer): assert opentracing.tracer == tracer # sanity check that patch worked span = tracer.start_span('root') with span: # use span as context manager so that it's always finished wrapper = SpanWrapper(span=span) with context_provider.request_context(wrapper): response_future = tchannel.raw( service='test-client', hostport=mock_server.hostport, endpoint='endpoint1', headers=mock_server.hostport, trace=False, ) response = yield response_future assert span.context.trace_id == long(response.body)
def test_both_connection_change_callback(): client = TChannel('client') with mock.patch.object(Peer, '_on_conn_change') as mock_conn_change: server = TChannel('server') server.listen() @server.raw.register def hello(request): return 'hi' yield client.raw(hostport=server.hostport, body='work', endpoint='hello', service='server') assert mock_conn_change.call_count == 6
def test_both_connection_change_callback(): client = TChannel('client') with mock.patch.object(Peer, '_on_conn_change') as mock_conn_change: server = TChannel('server') server.listen() @server.raw.register def hello(request): return 'hi' yield client.raw( hostport=server.hostport, body='work', endpoint='hello', service='server' ) assert mock_conn_change.call_count == 6
def test_tchannel_call_request_fragment(mock_server, arg2, arg3): endpoint = b'tchannelpeertest' mock_server.expect_call(endpoint).and_write(headers=endpoint, body=arg3) tchannel = TChannel(name='test') response = yield tchannel.raw( service='test-service', hostport=mock_server.hostport, endpoint=endpoint, headers=arg2, body=arg3, ) assert response.headers == endpoint assert response.body == arg3 assert response.transport.scheme == 'raw'
def test_peer_connection_failure_exhausted_peers(): # If we run out of healthy peers while trying to connect, raise # NoAvailablePeerError. servers = [TChannel('server-%d' % n) for n in xrange(10)] for server in servers: server.listen() known_peers = [server.hostport for server in servers] client = TChannel('client', known_peers=known_peers) for server in servers: # TODO New TChannel doesn't have close() and old one doesn't call # stop() on server. server._dep_tchannel._server.stop() with pytest.raises(NoAvailablePeerError): yield client.raw('server', 'hello', 'foo')
def test_unexpected_error(error, msg): class Handler(object): def handle(self, request): raise error inbound = TChannelInbound(TChannel('%s-server' % __name__)) inbound.start(Handler()) client = TChannel('%s-client' % __name__) with pytest.raises(TChUnexpectedError) as e: yield client.raw( service='service', endpoint='procedure', hostport=inbound.hostport, timeout=10, # seconds ) e = e.value assert str(e) == msg
def test_bad_request_error(req, msg): # we need to mock yarpc.transport.tchannel._to_request to produce # faulty yarpc.transport.Request objects because its not possible to # send bad requests with the tchannel client library. from yarpc.transport.tchannel import inbound as tch_inbound_module allow(tch_inbound_module)._to_request.and_return(req) inbound = TChannelInbound(TChannel('%s-server' % __name__)) inbound.start(None) client = TChannel('%s-client' % __name__) with pytest.raises(TChBadRequestError) as e: yield client.raw( service='service', endpoint='procedure', hostport=inbound.hostport, timeout=10, # seconds ) e = e.value assert str(e) == msg
def test_peer_connection_network_failure(): # Network errors in connecting to a peer must be retried with a different # peer. healthy = TChannel('healthy-server') healthy.listen() unhealthy = TChannel('unhealthy-server') unhealthy.listen() # register the endpoint on the healthy host only to ensure that the # request wasn't made to the unhealthy one. @healthy.raw.register('hello') def endpoint(request): return 'world' known_peers = [healthy.hostport, unhealthy.hostport] client = TChannel('client', known_peers=known_peers) with mock.patch.object(tpeer.PeerGroup, 'choose') as mock_choose: def fake_choose(*args, **kwargs): if mock_choose.call_count == 1: # First choose the unhealthy host. hostport = unhealthy.hostport else: hostport = healthy.hostport # TODO need access to peers in new TChannel return client._dep_tchannel.peers.get(hostport) mock_choose.side_effect = fake_choose # TODO New TChannel doesn't have close() and old one doesn't call # stop() on server. unhealthy._dep_tchannel._server.stop() resp = yield client.raw('server', 'hello', 'foo') assert resp.body == 'world'
def test_call_should_get_response(): # Given this test server: server = TChannel(name='server') @server.raw.register def endpoint(request): assert isinstance(request, Request) assert request.headers == 'req headers' assert request.body == 'req body' return Response('resp body', headers='resp headers') server.listen() # Make a call: tchannel = TChannel(name='client') resp = yield tchannel.raw( service='server', endpoint='endpoint', headers='req headers', body='req body', hostport=server.hostport, ) # verify response assert isinstance(resp, Response) assert resp.headers == 'resp headers' assert resp.body == 'resp body' # verify response transport headers assert isinstance(resp.transport, TransportHeaders) assert resp.transport.scheme == schemes.RAW assert resp.transport.failure_domain is None
def test_zipkin_trace(trace_server): endpoint = b'endpoint1' zipkin_tracer = ZipkinTraceHook(dst=trace_buf) tchannel = TChannel(name='test') tchannel.hooks.register(zipkin_tracer) hostport = 'localhost:%d' % trace_server.port response = yield tchannel.raw( service='test-client', hostport=hostport, endpoint=endpoint, headers=hostport, trace=True, ) header = response.headers body = response.body assert header == "from handler1" assert body == "from handler2" traces = [] for trace in trace_buf.getvalue().split("\n"): if trace: traces.append(json.loads(trace)) parent_span_id = object() trace_id = traces[0][0][u'trace_id'] assert traces for trace in traces: assert trace_id == trace[0][u'trace_id'] if trace[0][u'name'] == u'endpoint2': parent_span_id = trace[0][u'parent_span_id'] else: span_id = trace[0][u'span_id'] assert parent_span_id == span_id
def test_call_should_get_response(): # Given this test server: server = TChannel(name='server') @server.raw.register def endpoint(request): assert isinstance(request, Request) assert request.headers == b'req headers' assert request.body == b'req body' return Response('resp body', headers='resp headers') server.listen() # Make a call: tchannel = TChannel(name='client') resp = yield tchannel.raw( service='server', endpoint='endpoint', headers='req headers', body='req body', hostport=server.hostport, ) # verify response assert isinstance(resp, Response) assert resp.headers == b'resp headers' assert resp.body == b'resp body' # verify response transport headers assert isinstance(resp.transport, TransportHeaders) assert resp.transport.scheme == schemes.RAW assert resp.transport.failure_domain is None
def test_register_should_work_with_different_endpoint(): # Given this test server: server = TChannel(name='server') @server.raw.register('foo') def endpoint(request): return 'resp body' server.listen() # Make a call: tchannel = TChannel(name='client') resp = yield tchannel.raw( service='server', endpoint='foo', hostport=server.hostport, ) assert resp.body == 'resp body'
def test_register_should_work_with_different_endpoint(): # Given this test server: server = TChannel(name='server') @server.raw.register('foo') def endpoint(request): return 'resp body' server.listen() # Make a call: tchannel = TChannel(name='client') resp = yield tchannel.raw( service='server', endpoint='foo', hostport=server.hostport, ) assert resp.body == b'resp body'
def test_endpoint_not_found(mock_server): tchannel = TChannel(name="test") with pytest.raises(BadRequestError): yield tchannel.raw(service="test-server", endpoint="fooo", hostport=mock_server.hostport)
def test_trace_mixed(endpoint, transport, expect_spans, http_patchers, tracer, mock_server, app, http_server, base_url, http_client): """ Main TChannel-OpenTracing integration test, using basictracer as implementation of OpenTracing API. The main logic of this test is as follows: 1. Start a new trace with a root span 2. Store a random value in the baggage 3. Call the first service at the endpoint from `endpoint` parameter. The first service is either tchannel or http, depending on the value if `transport` parameter. 4. The first service calls the second service using pre-defined logic that depends on the endpoint invoked on the first service. 5. The second service accesses the tracing span and returns the value of the baggage item as the response. 6. The first service responds with the value from the second service. 7. The main test validates that the response is equal to the original random value of the baggage, proving trace & baggage propagation. 8. The test also validates that all spans have been finished and recorded, and that they all have the same trace ID. We expect 5 spans to be created from each test run: * top-level (root) span started in the test * client span (calling service-1) * service-1 server span * service-1 client span (calling service-2) * service-2 server span :param endpoint: name of the endpoint to call on the first service :param transport: type of the first service: tchannel or http :param expect_spans: number of spans we expect to be generated :param http_patchers: monkey-patching of tornado AsyncHTTPClient :param tracer: a concrete implementation of OpenTracing Tracer :param mock_server: tchannel server (from conftest.py) :param app: tornado.web.Application fixture :param http_server: http server (provided by pytest-tornado) :param base_url: address of http server (provided by pytest-tornado) :param http_client: Tornado's AsyncHTTPClient (provided by pytest-tornado) """ context_provider = OpenTracingRequestContextProvider() hook = OpenTracingHook(tracer=tracer, context_provider=context_provider) mock_server.tchannel.context_provider = context_provider mock_server.tchannel.hooks.register(hook) register(mock_server.tchannel, http_client=http_client, base_url=base_url) tchannel = TChannel(name='test', context_provider=context_provider) tchannel.hooks.register(hook) app.add_handlers(".*$", [ (r"/", HttpHandler, {'client_channel': tchannel}) ]) with mock.patch('opentracing.tracer', tracer): assert opentracing.tracer == tracer # sanity check that patch worked span = tracer.start_span('root') baggage = 'from handler3 %d' % time.time() span.set_baggage_item(BAGGAGE_KEY, baggage) with span: # use span as context manager so that it's always finished wrapper = SpanWrapper(span=span) response_future = None with context_provider.request_context(wrapper): if transport == 'tchannel': response_future = tchannel.raw( service='test-client', hostport=mock_server.hostport, endpoint=endpoint, headers=mock_server.hostport, trace=False, ) elif transport == 'http': response_future = http_client.fetch( request=HTTPRequest( url='%s%s' % (base_url, endpoint), method='POST', body=mock_server.hostport, ) ) else: raise NotImplementedError( 'unknown transport %s' % transport) response = yield response_future body = response.body assert body == baggage # Sometimes the test runs into weird race condition where the # after_send_response() hook is executed, but the span is not yet # recorded. To prevent flaky test runs we check and wait until # all spans are recorded, for up to 1 second. for i in range(0, 1000): spans = tracer.recorder.get_spans() if len(spans) == expect_spans: break yield tornado.gen.sleep(0.001) # yield execution and sleep for 1ms spans = tracer.recorder.get_spans() assert expect_spans == len(spans) # We expect all trace IDs in collected spans to be the same trace_id = spans[0].context.trace_id for i in range(1, len(spans)): assert trace_id == spans[i].context.trace_id, 'span #%d' % i