def test_proxy_disconnect_with_active_worker(container_factory, rabbit_manager, rabbit_config): """ Break the connection to rabbit while a service's queue consumer and rabbit while the service has an in-flight rpc request (i.e. it is waiting on a reply). """ # ExampleService is the target; ProxyService has the rpc_proxy; proxy_container = container_factory(ProxyService, rabbit_config) example_container = container_factory(ExampleService, rabbit_config) proxy_container.start() # get proxyservice's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 proxy_consumer_conn = connections[0]['name'] example_container.start() # there should now be two connections: # 1. the queue consumer from proxyservice # 2. the queue consumer from exampleservice connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 2 # disconnect proxyservice's queue consumer while its request is in-flight eventlet.spawn(disconnect_on_event, rabbit_manager, proxy_consumer_conn) with entrypoint_hook(proxy_container, 'entrypoint') as entrypoint: # we should receive a response after reconnection assert entrypoint('hello') == 'hello' connections = get_rabbit_connections(vhost, rabbit_manager) assert proxy_consumer_conn not in [conn['name'] for conn in connections]
def test_service_disconnect_with_active_async_worker( container_factory, rabbit_manager, rabbit_config): """ Break the connection between a service's queue consumer and rabbit while the service has an active async worker (e.g. event handler). """ container = container_factory(ExampleService, rabbit_config) container.start() # get the service's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 queue_consumer_conn = connections[0]['name'] # disconnect the service's queue consumer while it's running the worker eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn) # dispatch an event data = uuid.uuid4().hex dispatch = event_dispatcher(rabbit_config) dispatch('srcservice', 'exampleevent', data) # `handle` will have been called twice with the same the `data`, because # rabbit will have redelivered the un-ack'd message from the first call def event_handled_twice(): assert handle_called.call_args_list == [call(data), call(data)] assert_stops_raising(event_handled_twice) connections = get_rabbit_connections(vhost, rabbit_manager) assert queue_consumer_conn not in [conn['name'] for conn in connections]
def test_service_disconnect_with_active_async_worker(container_factory, rabbit_manager, rabbit_config): """ Break the connection between a service's queue consumer and rabbit while the service has an active async worker (e.g. event handler). """ container = container_factory(ExampleService, rabbit_config) container.start() # get the service's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 queue_consumer_conn = connections[0]['name'] # disconnect the service's queue consumer while it's running the worker eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn) # dispatch an event data = uuid.uuid4().hex with event_dispatcher('srcservice', rabbit_config) as dispatch: dispatch(ExampleEvent(data)) # `handle` will have been called twice with the same the `data`, because # rabbit will have redelivered the un-ack'd message from the first call def event_handled_twice(): assert handle_called.call_args_list == [call(data), call(data)] assert_stops_raising(event_handled_twice) connections = get_rabbit_connections(vhost, rabbit_manager) assert queue_consumer_conn not in [conn['name'] for conn in connections]
def test_service_disconnect_with_active_rpc_worker_via_service_proxy( logger, container_factory, rabbit_manager, rabbit_config): """ Break the connection between a service's queue consumer and rabbit while the service has an active rpc worker (i.e. response required). Make the rpc call from a nameko service. We expect the service to see the duplicate response and discard it. """ # ExampleService is the target; ProxyService has the rpc_proxy; proxy_container = container_factory(ProxyService, rabbit_config) service_container = container_factory(ExampleService, rabbit_config) service_container.start() # get exampleservice's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 service_consumer_conn = connections[0]['name'] proxy_container.start() # there should now be two connections: # 1. the queue consumer from proxyservice # 2. the queue consumer from exampleservice connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 2 # disconnect exampleservice's queue consumer while it's running the worker eventlet.spawn(disconnect_on_event, rabbit_manager, service_consumer_conn) # we should receive the response from the first call # the service rpc_proxy will receive and discard the response from the # second call arg = uuid.uuid4().hex with entrypoint_hook(proxy_container, 'entrypoint') as entrypoint: # we should receive a response after reconnection assert entrypoint(arg) == arg def duplicate_response_received(): correlation_warning = call("Unknown correlation id: %s", ANY) assert correlation_warning in logger.debug.call_args_list assert_stops_raising(duplicate_response_received) # `method` will have been called twice with the same the `arg`, because # rabbit will have redelivered the un-ack'd message from the first call def method_called_twice(): assert method_called.call_args_list == [call(arg), call(arg)] assert_stops_raising(method_called_twice) connections = get_rabbit_connections(vhost, rabbit_manager) assert service_consumer_conn not in [conn['name'] for conn in connections]
def test_runner_factory( testdir, plugin_options, rabbit_config, rabbit_manager ): testdir.makepyfile( """ from nameko.rpc import rpc from nameko.standalone.rpc import ServiceRpcProxy class ServiceX(object): name = "x" @rpc def method(self): return "OK" def test_runner(runner_factory, rabbit_config): runner = runner_factory(rabbit_config, ServiceX) runner.start() with ServiceRpcProxy("x", rabbit_config) as proxy: assert proxy.method() == "OK" """ ) result = testdir.runpytest(*plugin_options) assert result.ret == 0 vhost = rabbit_config['vhost'] assert get_rabbit_connections(vhost, rabbit_manager) == []
def rabbit_config(request, rabbit_manager): from kombu import pools from nameko.testing.utils import ( reset_rabbit_vhost, reset_rabbit_connections, get_rabbit_connections, get_rabbit_config, ) amqp_uri = request.config.getoption("AMQP_URI") conf = get_rabbit_config(amqp_uri) reset_rabbit_connections(conf["vhost"], rabbit_manager) reset_rabbit_vhost(conf["vhost"], conf["username"], rabbit_manager) yield conf pools.reset() # close connections in pools # raise a runtime error if the test leaves any connections lying around connections = get_rabbit_connections(conf["vhost"], rabbit_manager) open_connections = [conn for conn in connections if conn["state"] != "closed"] if open_connections: count = len(open_connections) raise RuntimeError("{} rabbit connection(s) left open.".format(count))
def test_kill_closes_connections(rabbit_manager, rabbit_config): container = Mock() container.config = rabbit_config container.max_workers = 1 container.spawn_managed_thread = spawn_thread queue_consumer = QueueConsumer() queue_consumer.bind("queue_consumer", container) class Handler(object): queue = ham_queue def handle_message(self, body, message): pass queue_consumer.register_provider(Handler()) queue_consumer.start() # kill should close all connections queue_consumer.kill() # no connections should remain for our vhost vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) if connections: for connection in connections: assert connection['vhost'] != vhost
def test_kill_closes_connections(rabbit_manager, rabbit_config, mock_container): container = mock_container container.shared_extensions = {} container.config = rabbit_config container.max_workers = 1 container.spawn_managed_thread = spawn_managed_thread queue_consumer = QueueConsumer().bind(container) queue_consumer.setup() class Handler(object): queue = ham_queue def handle_message(self, body, message): pass # pragma: no cover queue_consumer.register_provider(Handler()) queue_consumer.start() # kill should close all connections queue_consumer.kill() # no connections should remain for our vhost vhost = rabbit_config["vhost"] connections = get_rabbit_connections(vhost, rabbit_manager) if connections: # pragma: no cover for connection in connections: assert connection["vhost"] != vhost
def test_kill_closes_connections(rabbit_manager, rabbit_config, mock_container): container = mock_container container.shared_extensions = {} container.config = rabbit_config container.max_workers = 1 container.spawn_managed_thread = spawn_managed_thread queue_consumer = QueueConsumer().bind(container) queue_consumer.setup() class Handler(object): queue = ham_queue def handle_message(self, body, message): pass # pragma: no cover queue_consumer.register_provider(Handler()) queue_consumer.start() # kill should close all connections queue_consumer.kill() # no connections should remain for our vhost vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) if connections: # pragma: no cover for connection in connections: assert connection['vhost'] != vhost
def test_runner_factory(testdir, plugin_options, rabbit_config, rabbit_manager): testdir.makepyfile(""" from nameko.rpc import rpc from nameko.standalone.rpc import ServiceRpcProxy class ServiceX(object): name = "x" @rpc def method(self): return "OK" def test_runner(runner_factory, rabbit_config): runner = runner_factory(rabbit_config, ServiceX) runner.start() with ServiceRpcProxy("x", rabbit_config) as proxy: assert proxy.method() == "OK" """) result = testdir.runpytest(*plugin_options) assert result.ret == 0 vhost = rabbit_config['vhost'] assert get_rabbit_connections(vhost, rabbit_manager) == []
def rabbit_config(request, rabbit_manager): from kombu import pools from nameko.testing.utils import (reset_rabbit_vhost, reset_rabbit_connections, get_rabbit_connections, get_rabbit_config) amqp_uri = request.config.getoption('AMQP_URI') conf = get_rabbit_config(amqp_uri) reset_rabbit_connections(conf['vhost'], rabbit_manager) reset_rabbit_vhost(conf['vhost'], conf['username'], rabbit_manager) yield conf pools.reset() # close connections in pools # raise a runtime error if the test leaves any connections lying around connections = get_rabbit_connections(conf['vhost'], rabbit_manager) open_connections = [ conn for conn in connections if conn['state'] != "closed" ] if open_connections: count = len(open_connections) raise RuntimeError("{} rabbit connection(s) left open.".format(count))
def test_service_disconnect_with_active_rpc_worker(container_factory, rabbit_manager, rabbit_config): """ Break the connection between a service's queue consumer and rabbit while the service has an active rpc worker (i.e. response required). """ container = container_factory(ExampleService, rabbit_config) container.start() # get the service's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 queue_consumer_conn = connections[0]['name'] # create a standalone RPC proxy towards the target service rpc_proxy = RpcProxy('exampleservice', rabbit_config) proxy = rpc_proxy.start() # there should now be two connections: # 1. the queue consumer from the target service # 2. the queue consumer in the standalone rpc proxy connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 2 # disconnect the service's queue consumer while it's running a worker eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn) # we should receive the response from the first call # the standalone RPC proxy will stop listening as soon as it receives # a reply, so the duplicate response is discarded arg = uuid.uuid4().hex assert proxy.method(arg) == arg # `method` will have been called twice with the same the `arg`, because # rabbit will have redelivered the un-ack'd message from the first call def method_called_twice(): assert method_called.call_args_list == [call(arg), call(arg)] assert_stops_raising(method_called_twice) connections = get_rabbit_connections(vhost, rabbit_manager) assert queue_consumer_conn not in [conn['name'] for conn in connections] rpc_proxy.stop()
def test_service_disconnect_with_active_rpc_worker( container_factory, rabbit_manager, rabbit_config): """ Break the connection between a service's queue consumer and rabbit while the service has an active rpc worker (i.e. response required). """ container = container_factory(ExampleService, rabbit_config) container.start() # get the service's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 queue_consumer_conn = connections[0]['name'] # create a standalone RPC proxy towards the target service rpc_proxy = ServiceRpcProxy('exampleservice', rabbit_config) proxy = rpc_proxy.start() # there should now be two connections: # 1. the queue consumer from the target service # 2. the queue consumer in the standalone rpc proxy connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 2 # disconnect the service's queue consumer while it's running a worker eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn) # we should receive the response from the first call # the standalone RPC proxy will stop listening as soon as it receives # a reply, so the duplicate response is discarded arg = uuid.uuid4().hex assert proxy.method(arg) == arg # `method` will have been called twice with the same the `arg`, because # rabbit will have redelivered the un-ack'd message from the first call def method_called_twice(): assert method_called.call_args_list == [call(arg), call(arg)] assert_stops_raising(method_called_twice) connections = get_rabbit_connections(vhost, rabbit_manager) assert queue_consumer_conn not in [conn['name'] for conn in connections] rpc_proxy.stop()
def test_disconnect_with_pending_reply(container_factory, rabbit_manager, rabbit_config): example_container = container_factory(ExampleService, rabbit_config) example_container.start() vhost = rabbit_config['vhost'] # get exampleservice's queue consumer connection while we know it's the # only active connection connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 container_connection = connections[0] with ServiceRpcProxy('exampleservice', rabbit_config) as proxy: connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 2 proxy_connection = [ conn for conn in connections if conn != container_connection ][0] def disconnect_once(self): if hasattr(disconnect_once, 'called'): return disconnect_once.called = True rabbit_manager.delete_connection(proxy_connection['name']) with patch.object(ExampleService, 'callback', disconnect_once): async = proxy.method.call_async('hello') # if disconnecting while waiting for a reply, call fails with pytest.raises(RpcConnectionError): proxy.method('hello') # the failure above also has to consider any other pending calls a # failure, since the reply may have been sent while the queue was # gone (deleted on disconnect, and not added until re-connect) with pytest.raises(RpcConnectionError): async .result() # proxy should work again afterwards assert proxy.method('hello') == 'hello'
def test_disconnect_with_pending_reply( container_factory, rabbit_manager, rabbit_config): example_container = container_factory(ExampleService, rabbit_config) example_container.start() vhost = rabbit_config['vhost'] # get exampleservice's queue consumer connection while we know it's the # only active connection connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 container_connection = connections[0] with ServiceRpcProxy('exampleservice', rabbit_config) as proxy: connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 2 proxy_connection = [ conn for conn in connections if conn != container_connection][0] def disconnect_once(self): if hasattr(disconnect_once, 'called'): return disconnect_once.called = True rabbit_manager.delete_connection(proxy_connection['name']) with patch.object(ExampleService, 'callback', disconnect_once): async = proxy.method.call_async('hello') # if disconnecting while waiting for a reply, call fails with pytest.raises(RpcConnectionError): proxy.method('hello') # the failure above also has to consider any other pending calls a # failure, since the reply may have been sent while the queue was # gone (deleted on disconnect, and not added until re-connect) with pytest.raises(RpcConnectionError): async.result() # proxy should work again afterwards assert proxy.method('hello') == 'hello'
def test_get_rabbit_connections(): vhost = "vhost_name" connections = [{ 'vhost': vhost, 'key': 'value' }, { 'vhost': 'unlikely_vhost_name', 'key': 'value' }] rabbit_manager = Mock() rabbit_manager.get_connections.return_value = connections vhost_conns = [connections[0]] assert get_rabbit_connections(vhost, rabbit_manager) == vhost_conns rabbit_manager.get_connections.return_value = None assert get_rabbit_connections(vhost, rabbit_manager) == []
def test_proxy_disconnect_with_active_worker(container_factory, rabbit_manager, rabbit_config): """ Break the connection to rabbit while a service's queue consumer and rabbit while the service has an in-flight rpc request (i.e. it is waiting on a reply). """ # ExampleService is the target; ProxyService has the rpc_proxy; proxy_container = container_factory(ProxyService, rabbit_config) example_container = container_factory(ExampleService, rabbit_config) proxy_container.start() # get proxyservice's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 proxy_consumer_conn = connections[0]['name'] example_container.start() # there should now be two connections: # 1. the queue consumer from proxyservice # 2. the queue consumer from exampleservice connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 2 # disconnect proxyservice's queue consumer while its request is in-flight eventlet.spawn(disconnect_on_event, rabbit_manager, proxy_consumer_conn) with entrypoint_hook(proxy_container, 'retry') as retry: # if disconnecting while waiting for a reply, call fails # fail, then success assert retry('hello') == [ (RpcConnectionError, 'Disconnected while waiting for reply'), 'duplicate-call-result', ] connections = get_rabbit_connections(vhost, rabbit_manager) assert proxy_consumer_conn not in [conn['name'] for conn in connections]
def test_proxy_disconnect_with_active_worker( container_factory, rabbit_manager, rabbit_config): """ Break the connection to rabbit while a service's queue consumer and rabbit while the service has an in-flight rpc request (i.e. it is waiting on a reply). """ # ExampleService is the target; ProxyService has the rpc_proxy; proxy_container = container_factory(ProxyService, rabbit_config) example_container = container_factory(ExampleService, rabbit_config) proxy_container.start() # get proxyservice's queue consumer connection while we know it's the # only active connection vhost = rabbit_config['vhost'] connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 proxy_consumer_conn = connections[0]['name'] example_container.start() # there should now be two connections: # 1. the queue consumer from proxyservice # 2. the queue consumer from exampleservice connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 2 # disconnect proxyservice's queue consumer while its request is in-flight eventlet.spawn(disconnect_on_event, rabbit_manager, proxy_consumer_conn) with entrypoint_hook(proxy_container, 'retry') as retry: # if disconnecting while waiting for a reply, call fails # fail, then success assert retry('hello') == [ (RpcConnectionError, 'Disconnected while waiting for reply'), 'duplicate-call-result', ] connections = get_rabbit_connections(vhost, rabbit_manager) assert proxy_consumer_conn not in [conn['name'] for conn in connections]
def check_connections(): """ Raise a runtime error if the test leaves any connections open. Allow a few retries because the rabbit api is eventually consistent. """ connections = get_rabbit_connections(conf['vhost'], rabbit_manager) open_connections = [ conn for conn in connections if conn['state'] != "closed" ] if open_connections: count = len(open_connections) names = ", ".join(conn['name'] for conn in open_connections) raise RuntimeError( "{} rabbit connection(s) left open: {}".format(count, names))
def rabbit_config(request, rabbit_manager): import random import string from kombu import pools from six.moves.urllib.parse import urlparse # pylint: disable=E0401 from nameko.testing.utils import get_rabbit_connections amqp_uri = request.config.getoption('AMQP_URI') uri = urlparse(amqp_uri) username = uri.username vhost = uri.path[1:] use_random_vost = (vhost == ":random:") if use_random_vost: vhost = "test_{}".format( "".join(random.choice(string.ascii_lowercase) for _ in range(10)) ) amqp_uri = "{}://{}/{}".format(uri.scheme, uri.netloc, vhost) rabbit_manager.create_vhost(vhost) rabbit_manager.set_vhost_permissions(vhost, username, '.*', '.*', '.*') conf = { 'AMQP_URI': amqp_uri, 'username': username, 'vhost': vhost } yield conf pools.reset() # close connections in pools # raise a runtime error if the test leaves any connections lying around try: connections = get_rabbit_connections(conf['vhost'], rabbit_manager) open_connections = [ conn for conn in connections if conn['state'] != "closed" ] if open_connections: count = len(open_connections) names = ", ".join(conn['name'] for conn in open_connections) raise RuntimeError( "{} rabbit connection(s) left open: {}".format(count, names)) finally: if use_random_vost: rabbit_manager.delete_vhost(vhost)
def rabbit_config(request, rabbit_manager): amqp_uri = request.config.getoption('AMQP_URI') conf = get_rabbit_config(amqp_uri) reset_rabbit_connections(conf['vhost'], rabbit_manager) reset_rabbit_vhost(conf['vhost'], conf['username'], rabbit_manager) yield conf pools.reset() # close connections in pools # raise a runtime error if the test leaves any connections lying around connections = get_rabbit_connections(conf['vhost'], rabbit_manager) if connections: count = len(connections) raise RuntimeError("{} rabbit connection(s) left open.".format(count))
def rabbit_config(request, rabbit_manager): amqp_uri = request.config.getoption('AMQP_URI') conf = {'AMQP_URI': amqp_uri} uri = urlparse(amqp_uri) vhost = uri.path[1:].replace('/', '%2F') username = uri.username conf['vhost'] = vhost conf['username'] = username reset_rabbit_connections(vhost, rabbit_manager) reset_rabbit_vhost(vhost, username, rabbit_manager) yield conf pools.reset() # close connections in pools # raise a runtime error if the test leaves any connections lying around connections = get_rabbit_connections(vhost, rabbit_manager) if connections: count = len(connections) raise RuntimeError("{} rabbit connection(s) left open.".format(count))
def check_connections_closed(): connections = get_rabbit_connections(vhost, rabbit_manager) if connections: # pragma: no cover for connection in connections: assert connection['vhost'] != vhost
def test_disconnect_with_pending_reply(container_factory, rabbit_manager, rabbit_config): block = Event() class ExampleService(object): name = "exampleservice" def hook(self): pass # pragma: no cover @rpc def method(self, arg): self.hook() block.wait() return arg container = container_factory(ExampleService, rabbit_config) container.start() vhost = rabbit_config['vhost'] # get exampleservice's queue consumer connection while we know it's the # only active connection connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 1 container_connection = connections[0] with ServiceRpcProxy('exampleservice', rabbit_config) as proxy: # grab the proxy's connection too, the only other connection connections = get_rabbit_connections(vhost, rabbit_manager) assert len(connections) == 2 proxy_connection = [ conn for conn in connections if conn != container_connection ][0] counter = itertools.count(start=1) class ConnectionStillOpen(Exception): pass @retry(for_exceptions=ConnectionStillOpen, delay=0.2) def wait_for_connection_close(name): connections = get_rabbit_connections(vhost, rabbit_manager) for conn in connections: if conn['name'] == name: raise ConnectionStillOpen(name) # pragma: no cover def cb(args, kwargs, res, exc_info): # trigger a disconnection on the second call. # release running workers once the connection has been closed count = next(counter) if count == 2: rabbit_manager.delete_connection(proxy_connection['name']) wait_for_connection_close(proxy_connection['name']) block.send(True) return True # attach a callback to `hook` so we can close the connection # while there are requests in-flight with wait_for_call(ExampleService, 'hook', callback=cb): # make an async call that runs for some time async_call = proxy.method.call_async("hello") # make another call that will trigger the disconnection; # expect the blocking proxy to raise when the service reconnects with pytest.raises(RpcConnectionError): proxy.method("hello") # also expect the running call to raise, since the reply may have # been sent while the queue was gone (deleted on disconnect, and # not added until re-connect) with pytest.raises(RpcConnectionError): async_call.result() # proxy should work again afterwards assert proxy.method("hello") == "hello"
def wait_for_connection_close(name): connections = get_rabbit_connections(vhost, rabbit_manager) for conn in connections: if conn['name'] == name: raise ConnectionStillOpen(name) # pragma: no cover