def est_client_drop_few(): endpoint = random_ipc_endpoint() class MySrv(zerorpc.Server): def lolita(self): return 42 srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) client2 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) client3 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) assert client1.lolita() == 42 assert client2.lolita() == 42 gevent.sleep(TIME_FACTOR * 3) assert client3.lolita() == 42
def test_task_context_pushpull(): puller_ctx = zerorpc.Context() pusher_ctx = zerorpc.Context() puller_tracer = Tracer('[puller]') puller_ctx.register_middleware(puller_tracer) pusher_tracer = Tracer('[pusher]') pusher_ctx.register_middleware(pusher_tracer) trigger = gevent.event.Event() class Puller(object): def echo(self, msg): trigger.set() puller = zerorpc.Puller(Puller(), context=puller_ctx) puller.bind(endpoint) puller_task = gevent.spawn(puller.run) c = zerorpc.Pusher(context=pusher_ctx) c.connect(endpoint) trigger.clear() c.echo('hello') trigger.wait() puller.stop() puller_task.join() assert pusher_tracer._log == [ ('new', pusher_tracer.trace_id), ] assert puller_tracer._log == [ ('load', pusher_tracer.trace_id), ]
def test_client_drop_stream(): endpoint = random_ipc_endpoint() class MySrv(zerorpc.Server): @zerorpc.stream def iter(self): return xrange(500) srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): print 'grab iter' i = client1.iter() print 'consume some' assert list(next(i) for x in xrange(142)) == list(xrange(142)) print 'sleep 3s' gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join()
def test_task_context_relay(): endpoint1 = random_ipc_endpoint() endpoint2 = random_ipc_endpoint() srv_ctx = zerorpc.Context() srv_relay_ctx = zerorpc.Context() cli_ctx = zerorpc.Context() srv_tracer = Tracer('[server]') srv_ctx.register_middleware(srv_tracer) srv_relay_tracer = Tracer('[server_relay]') srv_relay_ctx.register_middleware(srv_relay_tracer) cli_tracer = Tracer('[client]') cli_ctx.register_middleware(cli_tracer) class Srv: def echo(self, msg): return msg srv = zerorpc.Server(Srv(), context=srv_ctx) srv.bind(endpoint1) srv_task = gevent.spawn(srv.run) c_relay = zerorpc.Client(context=srv_relay_ctx) c_relay.connect(endpoint1) class SrvRelay: def echo(self, msg): return c_relay.echo('relay' + msg) + 'relayed' srv_relay = zerorpc.Server(SrvRelay(), context=srv_relay_ctx) srv_relay.bind(endpoint2) srv_relay_task = gevent.spawn(srv_relay.run) c = zerorpc.Client(context=cli_ctx) c.connect(endpoint2) assert c.echo('hello') == 'relayhellorelayed' srv_relay.stop() srv.stop() srv_relay_task.join() srv_task.join() assert cli_tracer._log == [ ('new', cli_tracer.trace_id), ] assert srv_relay_tracer._log == [ ('load', cli_tracer.trace_id), ('reuse', cli_tracer.trace_id), ('reuse', cli_tracer.trace_id), ] assert srv_tracer._log == [ ('load', cli_tracer.trace_id), ('reuse', cli_tracer.trace_id), ]
def test_hook_client_before_request(): class ClientBeforeRequestMiddleware(object): def __init__(self): self.called = False def client_before_request(self, event): self.called = True self.method = event.name zero_ctx = zerorpc.Context() test_server = zerorpc.Server(EchoModule(), context=zero_ctx) test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) test_client = zerorpc.Client(context=zero_ctx) test_client.connect(endpoint) assert test_client.echo("test") == "echo: test" test_middleware = ClientBeforeRequestMiddleware() zero_ctx.register_middleware(test_middleware) assert test_middleware.called == False assert test_client.echo("test") == "echo: test" assert test_middleware.called == True assert test_middleware.method == 'echo' test_server.stop() test_server_task.join()
def test_hook_server_before_exec_stream(): zero_ctx = zerorpc.Context() test_server = zerorpc.Server(EchoModule(), context=zero_ctx) test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) test_client = zerorpc.Client() test_client.connect(endpoint) # Test without a middleware for echo in test_client.echoes("test"): assert echo == "echo: test" # Test with a middleware test_middleware = ServerBeforeExecMiddleware() zero_ctx.register_middleware(test_middleware) assert test_middleware.called == False it = test_client.echoes("test") assert test_middleware.called == True assert next(it) == "echo: test" for echo in it: assert echo == "echo: test" test_server.stop() test_server_task.join()
def test_resolve_endpoint_events(): c = zerorpc.Context() class Resolver(object): def resolve_endpoint(self, endpoint): if endpoint == 'some_service': return test_endpoint return endpoint class Srv(zerorpc.Server): def hello(self): print('heee') return 'world' srv = Srv(heartbeat=TIME_FACTOR * 1, context=c) if sys.version_info < (2, 7): assert_raises(zmq.ZMQError, srv.bind, 'some_service') else: with assert_raises(zmq.ZMQError): srv.bind('some_service') cnt = c.register_middleware(Resolver()) assert cnt == 1 srv.bind('some_service') gevent.spawn(srv.run) client = zerorpc.Client(heartbeat=TIME_FACTOR * 1, context=c) client.connect('some_service') assert client.hello() == 'world' client.close() srv.close()
def test_hook_server_before_exec_puller(): zero_ctx = zerorpc.Context() trigger = gevent.event.Event() endpoint = random_ipc_endpoint() echo_module = EchoModule(trigger) test_server = zerorpc.Puller(echo_module, context=zero_ctx) test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) test_client = zerorpc.Pusher() test_client.connect(endpoint) # Test without a middleware test_client.echo("test") trigger.wait(timeout=2) assert echo_module.last_msg == "echo: test" trigger.clear() # Test with a middleware test_middleware = ServerBeforeExecMiddleware() zero_ctx.register_middleware(test_middleware) assert test_middleware.called == False test_client.echo("test with a middleware") trigger.wait(timeout=2) assert echo_module.last_msg == "echo: test with a middleware" assert test_middleware.called == True test_server.stop() test_server_task.join()
def test_hook_server_after_exec_stream(): zero_ctx = zerorpc.Context() endpoint = random_ipc_endpoint() test_server = zerorpc.Server(EchoModule(), context=zero_ctx) test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) test_client = zerorpc.Client() test_client.connect(endpoint) # Test without a middleware for echo in test_client.echoes("test"): assert echo == "echo: test" # Test with a middleware test_middleware = ServerAfterExecMiddleware() zero_ctx.register_middleware(test_middleware) assert test_middleware.called == False it = test_client.echoes("test") assert next(it) == "echo: test" assert test_middleware.called == False for echo in it: assert echo == "echo: test" assert test_middleware.called == True assert test_middleware.request_event_name == 'echoes' assert test_middleware.reply_event_name == 'STREAM_DONE' test_server.stop() test_server_task.join()
def test_hook_server_after_exec_on_error_puller(): zero_ctx = zerorpc.Context() trigger = gevent.event.Event() endpoint = random_ipc_endpoint() echo_module = BrokenEchoModule(trigger) test_server = zerorpc.Puller(echo_module, context=zero_ctx) test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) test_client = zerorpc.Pusher() test_client.connect(endpoint) test_middleware = ServerAfterExecMiddleware() zero_ctx.register_middleware(test_middleware) assert test_middleware.called == False try: test_client.echo("test with a middleware") trigger.wait(timeout=2) except zerorpc.RemoteError: pass assert echo_module.last_msg == "Raise" assert test_middleware.called == False test_server.stop() test_server_task.join()
def test_msgpack_numpy(): import numpy as np context = zerorpc.Context() test_array = np.arange(10) event = zerorpc.Event(u'myevent', (test_array, ), {}, context=context) print(event) # note here that str is an unicode string in all Python version (thanks to # the builtin str import). assert isinstance(event.name, str) for key in event.header.keys(): assert isinstance(key, str) assert isinstance(event.header[u'message_id'], bytes) assert isinstance(event.header[u'v'], int) assert isinstance(event.args[0], np.ndarray) packed = event.pack(context.serializer) unpacked = zerorpc.Event.unpack(context.serializer, packed) print(unpacked) assert isinstance(event.name, str) for key in event.header.keys(): assert isinstance(key, str) assert isinstance(event.header[u'message_id'], bytes) assert isinstance(event.header[u'v'], int) assert isinstance(event.args[0], np.ndarray) assert np.array_equal(test_array, event.args[0])
def test_resolve_endpoint_events(): test_endpoint = random_ipc_endpoint() c = zerorpc.Context() class Resolver(): def resolve_endpoint(self, endpoint): if endpoint == 'some_service': return test_endpoint return endpoint class Srv(zerorpc.Server): def hello(self): print 'heee' return 'world' srv = Srv(heartbeat=1, context=c) with assert_raises(zmq.ZMQError): srv.bind('some_service') cnt = c.register_middleware(Resolver()) assert cnt == 1 srv.bind('some_service') gevent.spawn(srv.run) client = zerorpc.Client(heartbeat=1, context=c) client.connect('some_service') assert client.hello() == 'world' client.close() srv.close()
def test_hook_client_after_request_stream(): zero_ctx = zerorpc.Context() test_server = zerorpc.Server(EchoModule(), context=zero_ctx) test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) test_client = zerorpc.Client(context=zero_ctx) test_client.connect(endpoint) it = test_client.echoes("test") assert next(it) == "echo: test" for echo in it: assert echo == "echo: test" test_middleware = ClientAfterRequestMiddleware() zero_ctx.register_middleware(test_middleware) assert test_middleware.called == False it = test_client.echoes("test") assert next(it) == "echo: test" assert test_middleware.called == False for echo in it: assert echo == "echo: test" assert test_middleware.called == True assert test_middleware.retcode == 'STREAM_DONE' test_server.stop() test_server_task.join()
def test_server_inspect_exception_middleware_puller(): endpoint = random_ipc_endpoint() barrier = gevent.event.Event() middleware = InspectExceptionMiddleware(barrier) ctx = zerorpc.Context() ctx.register_middleware(middleware) module = Srv() server = zerorpc.Puller(module, context=ctx) server.bind(endpoint) gevent.spawn(server.run) client = zerorpc.Pusher() client.connect(endpoint) barrier.clear() client.echo( 'This is a test which should call the InspectExceptionMiddleware') barrier.wait(timeout=2) client.close() server.close() assert middleware.called is True
def test_server_inspect_exception_middleware_stream(): endpoint = random_ipc_endpoint() middleware = InspectExceptionMiddleware() ctx = zerorpc.Context() ctx.register_middleware(middleware) module = Srv() server = zerorpc.Server(module, context=ctx) server.bind(endpoint) gevent.spawn(server.run) client = zerorpc.Client() client.connect(endpoint) try: client.echo( 'This is a test which should call the InspectExceptionMiddleware') except zerorpc.exceptions.RemoteError as ex: assert ex.name == 'RuntimeError' client.close() server.close() assert middleware.called is True
def test_hook_client_after_request_timeout(): class ClientAfterRequestMiddleware(object): def __init__(self): self.called = False def client_after_request(self, req_event, rep_event, exception): self.called = True assert req_event is not None assert req_event.name == "timeout" assert rep_event is None zero_ctx = zerorpc.Context() test_middleware = ClientAfterRequestMiddleware() zero_ctx.register_middleware(test_middleware) test_server = zerorpc.Server(EchoModule(), context=zero_ctx) test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) test_client = zerorpc.Client(timeout=TIME_FACTOR * 1, context=zero_ctx) test_client.connect(endpoint) assert test_middleware.called == False try: test_client.timeout("test") except zerorpc.TimeoutExpired as ex: assert test_middleware.called == True assert "timeout" in ex.args[0] test_server.stop() test_server_task.join()
def test_hook_client_handle_remote_error_inspect(): class ClientHandleRemoteErrorMiddleware(object): def __init__(self): self.called = False def client_handle_remote_error(self, event): self.called = True test_middleware = ClientHandleRemoteErrorMiddleware() zero_ctx = zerorpc.Context() zero_ctx.register_middleware(test_middleware) test_server = zerorpc.Server(EchoModule(), context=zero_ctx) test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) test_client = zerorpc.Client(context=zero_ctx) test_client.connect(endpoint) assert test_middleware.called == False try: test_client.crash("test") except zerorpc.RemoteError as ex: assert test_middleware.called == True assert ex.name == "RuntimeError" test_server.stop() test_server_task.join()
def run_zrpcserver(self, host, port, server_started): # temporarily timing import statements to check FS times on HPC environs start = time.time() import gevent import zerorpc logger.info("Importing of gevent and zerorpc took %.3f seconds", time.time() - start) # Use a specific context; otherwise multiple servers on the same process # (only during tests) share the same Context.instance() which is global # to the process ctx = zerorpc.Context() self._zrpcserver = zerorpc.Server(self, context=ctx) # zmq needs an address, not a hostname endpoint = "tcp://%s:%d" % ( zmq_safe(host), port, ) self._zrpcserver.bind(endpoint) logger.info("Listening for RPC requests via ZeroRPC on %s", endpoint) server_started.set() runner = gevent.spawn(self._zrpcserver.run) stopper = gevent.spawn(self.stop_zrpcserver) gevent.joinall([runner, stopper]) ctx.destroy()
def test_hook_server_after_exec_puller(): zero_ctx = zerorpc.Context() trigger = gevent.event.Event() echo_module = EchoModule(trigger) test_server = zerorpc.Puller(echo_module, context=zero_ctx) test_server.bind(endpoint) test_server_task = gevent.spawn(test_server.run) test_client = zerorpc.Pusher() test_client.connect(endpoint) # Test without a middleware test_client.echo("test") trigger.wait(timeout=TIME_FACTOR * 2) assert echo_module.last_msg == "echo: test" trigger.clear() # Test with a middleware test_middleware = ServerAfterExecMiddleware() zero_ctx.register_middleware(test_middleware) assert test_middleware.called == False test_client.echo("test with a middleware") trigger.wait(timeout=TIME_FACTOR * 2) assert echo_module.last_msg == "echo: test with a middleware" assert test_middleware.called == True assert test_middleware.request_event_name == 'echo' assert test_middleware.reply_event_name is None test_server.stop() test_server_task.join()
def start(self): super(ZeroRPCClient, self).start() if not hasattr(self, "_context"): self._context = zerorpc.Context() self._own_context = True # One per remote host self._zrpcclient_acquisition_lock = threading.Lock() self._zrpcclients = {} self._zrpcclientthreads = []
def test_resolve_endpoint(): test_endpoint = random_ipc_endpoint() c = zerorpc.Context() def resolve(endpoint): if endpoint == 'titi': return test_endpoint return endpoint cnt = c.register_middleware({ 'resolve_endpoint': resolve }) print('registered_count:', cnt) assert cnt == 1 print('resolve titi:', c.hook_resolve_endpoint('titi')) assert c.hook_resolve_endpoint('titi') == test_endpoint print('resolve toto:', c.hook_resolve_endpoint('toto')) assert c.hook_resolve_endpoint('toto') == 'toto' class Resolver(object): def resolve_endpoint(self, endpoint): if endpoint == 'toto': return test_endpoint return endpoint cnt = c.register_middleware(Resolver()) print('registered_count:', cnt) assert cnt == 1 print('resolve titi:', c.hook_resolve_endpoint('titi')) assert c.hook_resolve_endpoint('titi') == test_endpoint print('resolve toto:', c.hook_resolve_endpoint('toto')) assert c.hook_resolve_endpoint('toto') == test_endpoint c2 = zerorpc.Context() print('resolve titi:', c2.hook_resolve_endpoint('titi')) assert c2.hook_resolve_endpoint('titi') == 'titi' print('resolve toto:', c2.hook_resolve_endpoint('toto')) assert c2.hook_resolve_endpoint('toto') == 'toto'
def test_task_context(): endpoint = random_ipc_endpoint() srv_ctx = zerorpc.Context() cli_ctx = zerorpc.Context() srv_tracer = Tracer('[server]') srv_ctx.register_middleware(srv_tracer) cli_tracer = Tracer('[client]') cli_ctx.register_middleware(cli_tracer) class Srv: def echo(self, msg): return msg @zerorpc.stream def stream(self): yield 42 srv = zerorpc.Server(Srv(), context=srv_ctx) srv.bind(endpoint) srv_task = gevent.spawn(srv.run) c = zerorpc.Client(context=cli_ctx) c.connect(endpoint) assert c.echo('hello') == 'hello' for x in c.stream(): assert x == 42 srv.stop() srv_task.join() assert cli_tracer._log == [ ('new', cli_tracer.trace_id), ('reuse', cli_tracer.trace_id), ] assert srv_tracer._log == [ ('load', cli_tracer.trace_id), ('reuse', cli_tracer.trace_id), ('load', cli_tracer.trace_id), ('reuse', cli_tracer.trace_id), ]
def test_client_hb_doesnt_linger_on_streaming(): class MySrv(zerorpc.Server): @zerorpc.stream def iter(self): return range(42) srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): assert list(client1.iter()) == list(range(42)) print('sleep 3s') gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join()
def test_client_hb_doesnt_linger_on_streaming(): endpoint = random_ipc_endpoint() class MySrv(zerorpc.Server): @zerorpc.stream def iter(self): return xrange(42) srv = MySrv(heartbeat=1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) def test_client(): assert list(client1.iter()) == list(xrange(42)) print 'sleep 3s' gevent.sleep(3) gevent.spawn(test_client).join()
def test_msgpack(): context = zerorpc.Context() event = zerorpc.Event('myevent', ('a',), context=context) print event assert type(event.name) == str for key in event.header.keys(): assert type(key) == str assert type(event.header['message_id']) == str assert type(event.args[0]) == str packed = event.pack() event = event.unpack(packed) print event assert type(event.name) == str for key in event.header.keys(): assert type(key) == str assert type(event.header['message_id']) == str assert type(event.args[0]) == str event = zerorpc.Event('myevent', (u'a',), context=context) print event assert type(event.name) == str for key in event.header.keys(): assert type(key) == str assert type(event.header['message_id']) == str assert type(event.args[0]) == unicode packed = event.pack() event = event.unpack(packed) print event assert type(event.name) == str for key in event.header.keys(): assert type(key) == str assert type(event.header['message_id']) == str assert type(event.args[0]) == unicode event = zerorpc.Event('myevent', (u'a', 'b'), context=context) print event assert type(event.name) == str for key in event.header.keys(): assert type(key) == str assert type(event.header['message_id']) == str assert type(event.args[0]) == unicode assert type(event.args[1]) == str packed = event.pack() event = event.unpack(packed) print event assert type(event.name) == str for key in event.header.keys(): assert type(key) == str assert type(event.header['message_id']) == str assert type(event.args[0]) == unicode assert type(event.args[1]) == str
def test_task_context_pubsub(): endpoint = random_ipc_endpoint() subscriber_ctx = zerorpc.Context() publisher_ctx = zerorpc.Context() subscriber_tracer = Tracer('[subscriber]') subscriber_ctx.register_middleware(subscriber_tracer) publisher_tracer = Tracer('[publisher]') publisher_ctx.register_middleware(publisher_tracer) trigger = gevent.event.Event() class Subscriber: def echo(self, msg): trigger.set() subscriber = zerorpc.Subscriber(Subscriber(), context=subscriber_ctx) subscriber.bind(endpoint) subscriber_task = gevent.spawn(subscriber.run) c = zerorpc.Publisher(context=publisher_ctx) c.connect(endpoint) trigger.clear() # We need this retry logic to wait that the subscriber.run coroutine starts # reading (the published messages will go to /dev/null until then). for attempt in xrange(0, 10): c.echo('pub...') if trigger.wait(0.2): break subscriber.stop() subscriber_task.join() assert publisher_tracer._log == [ ('new', publisher_tracer.trace_id), ] assert subscriber_tracer._log == [ ('load', publisher_tracer.trace_id), ]
def test_client_drop_empty_stream(): class MySrv(zerorpc.Server): @zerorpc.stream def iter(self): return [] srv = MySrv(heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) client1 = zerorpc.Client(endpoint, heartbeat=TIME_FACTOR * 1, context=zerorpc.Context()) def test_client(): print('grab iter') i = client1.iter() print('sleep 3s') gevent.sleep(TIME_FACTOR * 3) gevent.spawn(test_client).join()
def test_client_drop_empty_stream(): endpoint = random_ipc_endpoint() class MySrv(zerorpc.Server): @zerorpc.stream def iter(self): return [] srv = MySrv(heartbeat=1, context=zerorpc.Context()) srv.bind(endpoint) gevent.spawn(srv.run) client1 = zerorpc.Client(endpoint, heartbeat=1, context=zerorpc.Context()) def test_client(): print 'grab iter' i = client1.iter() print 'sleep 3s' gevent.sleep(3) gevent.spawn(test_client).join()
def test_task_context_pubsub(): subscriber_ctx = zerorpc.Context() publisher_ctx = zerorpc.Context() subscriber_tracer = Tracer('[subscriber]') subscriber_ctx.register_middleware(subscriber_tracer) publisher_tracer = Tracer('[publisher]') publisher_ctx.register_middleware(publisher_tracer) trigger = gevent.event.Event() class Subscriber(object): def echo(self, msg): trigger.set() subscriber = zerorpc.Subscriber(Subscriber(), context=subscriber_ctx) subscriber.bind(endpoint) subscriber_task = gevent.spawn(subscriber.run) c = zerorpc.Publisher(context=publisher_ctx) c.connect(endpoint) trigger.clear() # We need this retry logic to wait that the subscriber.run coroutine starts # reading (the published messages will go to /dev/null until then). while not trigger.is_set(): c.echo('pub...') if trigger.wait(TIME_FACTOR * 1): break subscriber.stop() subscriber_task.join() print(publisher_tracer._log) assert ('new', publisher_tracer.trace_id) in publisher_tracer._log print(subscriber_tracer._log) assert ('load', publisher_tracer.trace_id) in subscriber_tracer._log
def test_task_context_pubsub(): endpoint = random_ipc_endpoint() subscriber_ctx = zerorpc.Context() publisher_ctx = zerorpc.Context() subscriber_tracer = Tracer('[subscriber]') subscriber_ctx.register_middleware(subscriber_tracer) publisher_tracer = Tracer('[publisher]') publisher_ctx.register_middleware(publisher_tracer) trigger = gevent.event.Event() class Subscriber: def echo(self, msg): trigger.set() subscriber = zerorpc.Subscriber(Subscriber(), context=subscriber_ctx) subscriber.bind(endpoint) subscriber_task = gevent.spawn(subscriber.run) c = zerorpc.Publisher(context=publisher_ctx) c.connect(endpoint) trigger.clear() c.echo('pub...') trigger.wait() subscriber.stop() subscriber_task.join() assert publisher_tracer._log == [ ('new', publisher_tracer.trace_id), ] assert subscriber_tracer._log == [ ('load', publisher_tracer.trace_id), ]