def test_proxy_disconnect_with_active_worker(container_factory, rabbit_manager,
rabbit_config):
""" Break the connection to rabbit while a service's queue consumer and
rabbit while the service has an in-flight rpc request (i.e. it is waiting
on a reply).
"""
# ExampleService is the target; ProxyService has the rpc_proxy;
proxy_container = container_factory(ProxyService, rabbit_config)
example_container = container_factory(ExampleService, rabbit_config)
proxy_container.start()
# get proxyservice's queue consumer connection while we know it's the
# only active connection
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
proxy_consumer_conn = connections[0]['name']
example_container.start()
# there should now be two connections:
# 1. the queue consumer from proxyservice
# 2. the queue consumer from exampleservice
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
# disconnect proxyservice's queue consumer while its request is in-flight
eventlet.spawn(disconnect_on_event, rabbit_manager, proxy_consumer_conn)
with entrypoint_hook(proxy_container, 'entrypoint') as entrypoint:
# we should receive a response after reconnection
assert entrypoint('hello') == 'hello'
connections = get_rabbit_connections(vhost, rabbit_manager)
assert proxy_consumer_conn not in [conn['name'] for conn in connections]
def test_proxy_disconnect_with_active_worker(container_factory,
rabbit_manager, rabbit_config):
""" Break the connection to rabbit while a service's queue consumer and
rabbit while the service has an in-flight rpc request (i.e. it is waiting
on a reply).
"""
# ExampleService is the target; ProxyService has the rpc_proxy;
proxy_container = container_factory(ProxyService, rabbit_config)
example_container = container_factory(ExampleService, rabbit_config)
proxy_container.start()
# get proxyservice's queue consumer connection while we know it's the
# only active connection
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
proxy_consumer_conn = connections[0]['name']
example_container.start()
# there should now be two connections:
# 1. the queue consumer from proxyservice
# 2. the queue consumer from exampleservice
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
# disconnect proxyservice's queue consumer while its request is in-flight
eventlet.spawn(disconnect_on_event, rabbit_manager, proxy_consumer_conn)
with entrypoint_hook(proxy_container, 'entrypoint') as entrypoint:
# we should receive a response after reconnection
assert entrypoint('hello') == 'hello'
connections = get_rabbit_connections(vhost, rabbit_manager)
assert proxy_consumer_conn not in [conn['name'] for conn in connections]
def test_service_disconnect_with_active_async_worker(
container_factory, rabbit_manager, rabbit_config):
""" Break the connection between a service's queue consumer and rabbit
while the service has an active async worker (e.g. event handler).
"""
container = container_factory(ExampleService, rabbit_config)
container.start()
# get the service's queue consumer connection while we know it's the
# only active connection
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
queue_consumer_conn = connections[0]['name']
# disconnect the service's queue consumer while it's running the worker
eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn)
# dispatch an event
data = uuid.uuid4().hex
dispatch = event_dispatcher(rabbit_config)
dispatch('srcservice', 'exampleevent', data)
# `handle` will have been called twice with the same the `data`, because
# rabbit will have redelivered the un-ack'd message from the first call
def event_handled_twice():
assert handle_called.call_args_list == [call(data), call(data)]
assert_stops_raising(event_handled_twice)
connections = get_rabbit_connections(vhost, rabbit_manager)
assert queue_consumer_conn not in [conn['name'] for conn in connections]
def test_service_disconnect_with_active_async_worker(container_factory,
rabbit_manager,
rabbit_config):
""" Break the connection between a service's queue consumer and rabbit
while the service has an active async worker (e.g. event handler).
"""
container = container_factory(ExampleService, rabbit_config)
container.start()
# get the service's queue consumer connection while we know it's the
# only active connection
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
queue_consumer_conn = connections[0]['name']
# disconnect the service's queue consumer while it's running the worker
eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn)
# dispatch an event
data = uuid.uuid4().hex
with event_dispatcher('srcservice', rabbit_config) as dispatch:
dispatch(ExampleEvent(data))
# `handle` will have been called twice with the same the `data`, because
# rabbit will have redelivered the un-ack'd message from the first call
def event_handled_twice():
assert handle_called.call_args_list == [call(data), call(data)]
assert_stops_raising(event_handled_twice)
connections = get_rabbit_connections(vhost, rabbit_manager)
assert queue_consumer_conn not in [conn['name'] for conn in connections]
def test_service_disconnect_with_active_rpc_worker_via_service_proxy(
logger, container_factory, rabbit_manager, rabbit_config):
""" Break the connection between a service's queue consumer and rabbit
while the service has an active rpc worker (i.e. response required).
Make the rpc call from a nameko service. We expect the service to see
the duplicate response and discard it.
"""
# ExampleService is the target; ProxyService has the rpc_proxy;
proxy_container = container_factory(ProxyService, rabbit_config)
service_container = container_factory(ExampleService, rabbit_config)
service_container.start()
# get exampleservice's queue consumer connection while we know it's the
# only active connection
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
service_consumer_conn = connections[0]['name']
proxy_container.start()
# there should now be two connections:
# 1. the queue consumer from proxyservice
# 2. the queue consumer from exampleservice
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
# disconnect exampleservice's queue consumer while it's running the worker
eventlet.spawn(disconnect_on_event, rabbit_manager, service_consumer_conn)
# we should receive the response from the first call
# the service rpc_proxy will receive and discard the response from the
# second call
arg = uuid.uuid4().hex
with entrypoint_hook(proxy_container, 'entrypoint') as entrypoint:
# we should receive a response after reconnection
assert entrypoint(arg) == arg
def duplicate_response_received():
correlation_warning = call("Unknown correlation id: %s", ANY)
assert correlation_warning in logger.debug.call_args_list
assert_stops_raising(duplicate_response_received)
# `method` will have been called twice with the same the `arg`, because
# rabbit will have redelivered the un-ack'd message from the first call
def method_called_twice():
assert method_called.call_args_list == [call(arg), call(arg)]
assert_stops_raising(method_called_twice)
connections = get_rabbit_connections(vhost, rabbit_manager)
assert service_consumer_conn not in [conn['name'] for conn in connections]
def test_service_disconnect_with_active_rpc_worker_via_service_proxy(
logger, container_factory, rabbit_manager, rabbit_config):
""" Break the connection between a service's queue consumer and rabbit
while the service has an active rpc worker (i.e. response required).
Make the rpc call from a nameko service. We expect the service to see
the duplicate response and discard it.
"""
# ExampleService is the target; ProxyService has the rpc_proxy;
proxy_container = container_factory(ProxyService, rabbit_config)
service_container = container_factory(ExampleService, rabbit_config)
service_container.start()
# get exampleservice's queue consumer connection while we know it's the
# only active connection
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
service_consumer_conn = connections[0]['name']
proxy_container.start()
# there should now be two connections:
# 1. the queue consumer from proxyservice
# 2. the queue consumer from exampleservice
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
# disconnect exampleservice's queue consumer while it's running the worker
eventlet.spawn(disconnect_on_event, rabbit_manager, service_consumer_conn)
# we should receive the response from the first call
# the service rpc_proxy will receive and discard the response from the
# second call
arg = uuid.uuid4().hex
with entrypoint_hook(proxy_container, 'entrypoint') as entrypoint:
# we should receive a response after reconnection
assert entrypoint(arg) == arg
def duplicate_response_received():
correlation_warning = call("Unknown correlation id: %s", ANY)
assert correlation_warning in logger.debug.call_args_list
assert_stops_raising(duplicate_response_received)
# `method` will have been called twice with the same the `arg`, because
# rabbit will have redelivered the un-ack'd message from the first call
def method_called_twice():
assert method_called.call_args_list == [call(arg), call(arg)]
assert_stops_raising(method_called_twice)
connections = get_rabbit_connections(vhost, rabbit_manager)
assert service_consumer_conn not in [conn['name'] for conn in connections]
def test_runner_factory(
testdir, plugin_options, rabbit_config, rabbit_manager
):
testdir.makepyfile(
"""
from nameko.rpc import rpc
from nameko.standalone.rpc import ServiceRpcProxy
class ServiceX(object):
name = "x"
@rpc
def method(self):
return "OK"
def test_runner(runner_factory, rabbit_config):
runner = runner_factory(rabbit_config, ServiceX)
runner.start()
with ServiceRpcProxy("x", rabbit_config) as proxy:
assert proxy.method() == "OK"
"""
)
result = testdir.runpytest(*plugin_options)
assert result.ret == 0
vhost = rabbit_config['vhost']
assert get_rabbit_connections(vhost, rabbit_manager) == []
def rabbit_config(request, rabbit_manager):
from kombu import pools
from nameko.testing.utils import (
reset_rabbit_vhost,
reset_rabbit_connections,
get_rabbit_connections,
get_rabbit_config,
)
amqp_uri = request.config.getoption("AMQP_URI")
conf = get_rabbit_config(amqp_uri)
reset_rabbit_connections(conf["vhost"], rabbit_manager)
reset_rabbit_vhost(conf["vhost"], conf["username"], rabbit_manager)
yield conf
pools.reset() # close connections in pools
# raise a runtime error if the test leaves any connections lying around
connections = get_rabbit_connections(conf["vhost"], rabbit_manager)
open_connections = [conn for conn in connections if conn["state"] != "closed"]
if open_connections:
count = len(open_connections)
raise RuntimeError("{} rabbit connection(s) left open.".format(count))
def test_kill_closes_connections(rabbit_manager, rabbit_config):
container = Mock()
container.config = rabbit_config
container.max_workers = 1
container.spawn_managed_thread = spawn_thread
queue_consumer = QueueConsumer()
queue_consumer.bind("queue_consumer", container)
class Handler(object):
queue = ham_queue
def handle_message(self, body, message):
pass
queue_consumer.register_provider(Handler())
queue_consumer.start()
# kill should close all connections
queue_consumer.kill()
# no connections should remain for our vhost
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
if connections:
for connection in connections:
assert connection['vhost'] != vhost
def test_kill_closes_connections(rabbit_manager, rabbit_config, mock_container):
container = mock_container
container.shared_extensions = {}
container.config = rabbit_config
container.max_workers = 1
container.spawn_managed_thread = spawn_managed_thread
queue_consumer = QueueConsumer().bind(container)
queue_consumer.setup()
class Handler(object):
queue = ham_queue
def handle_message(self, body, message):
pass # pragma: no cover
queue_consumer.register_provider(Handler())
queue_consumer.start()
# kill should close all connections
queue_consumer.kill()
# no connections should remain for our vhost
vhost = rabbit_config["vhost"]
connections = get_rabbit_connections(vhost, rabbit_manager)
if connections: # pragma: no cover
for connection in connections:
assert connection["vhost"] != vhost
def test_kill_closes_connections(rabbit_manager, rabbit_config,
mock_container):
container = mock_container
container.shared_extensions = {}
container.config = rabbit_config
container.max_workers = 1
container.spawn_managed_thread = spawn_managed_thread
queue_consumer = QueueConsumer().bind(container)
queue_consumer.setup()
class Handler(object):
queue = ham_queue
def handle_message(self, body, message):
pass # pragma: no cover
queue_consumer.register_provider(Handler())
queue_consumer.start()
# kill should close all connections
queue_consumer.kill()
# no connections should remain for our vhost
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
if connections: # pragma: no cover
for connection in connections:
assert connection['vhost'] != vhost
def test_runner_factory(testdir, plugin_options, rabbit_config,
rabbit_manager):
testdir.makepyfile("""
from nameko.rpc import rpc
from nameko.standalone.rpc import ServiceRpcProxy
class ServiceX(object):
name = "x"
@rpc
def method(self):
return "OK"
def test_runner(runner_factory, rabbit_config):
runner = runner_factory(rabbit_config, ServiceX)
runner.start()
with ServiceRpcProxy("x", rabbit_config) as proxy:
assert proxy.method() == "OK"
""")
result = testdir.runpytest(*plugin_options)
assert result.ret == 0
vhost = rabbit_config['vhost']
assert get_rabbit_connections(vhost, rabbit_manager) == []
def rabbit_config(request, rabbit_manager):
from kombu import pools
from nameko.testing.utils import (reset_rabbit_vhost,
reset_rabbit_connections,
get_rabbit_connections,
get_rabbit_config)
amqp_uri = request.config.getoption('AMQP_URI')
conf = get_rabbit_config(amqp_uri)
reset_rabbit_connections(conf['vhost'], rabbit_manager)
reset_rabbit_vhost(conf['vhost'], conf['username'], rabbit_manager)
yield conf
pools.reset() # close connections in pools
# raise a runtime error if the test leaves any connections lying around
connections = get_rabbit_connections(conf['vhost'], rabbit_manager)
open_connections = [
conn for conn in connections if conn['state'] != "closed"
]
if open_connections:
count = len(open_connections)
raise RuntimeError("{} rabbit connection(s) left open.".format(count))
def test_kill_closes_connections(rabbit_manager, rabbit_config):
container = Mock()
container.config = rabbit_config
container.max_workers = 1
container.spawn_managed_thread = spawn_thread
queue_consumer = QueueConsumer()
queue_consumer.bind("queue_consumer", container)
class Handler(object):
queue = ham_queue
def handle_message(self, body, message):
pass
queue_consumer.register_provider(Handler())
queue_consumer.start()
# kill should close all connections
queue_consumer.kill()
# no connections should remain for our vhost
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
if connections:
for connection in connections:
assert connection['vhost'] != vhost
def test_service_disconnect_with_active_rpc_worker(container_factory,
rabbit_manager,
rabbit_config):
""" Break the connection between a service's queue consumer and rabbit
while the service has an active rpc worker (i.e. response required).
"""
container = container_factory(ExampleService, rabbit_config)
container.start()
# get the service's queue consumer connection while we know it's the
# only active connection
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
queue_consumer_conn = connections[0]['name']
# create a standalone RPC proxy towards the target service
rpc_proxy = RpcProxy('exampleservice', rabbit_config)
proxy = rpc_proxy.start()
# there should now be two connections:
# 1. the queue consumer from the target service
# 2. the queue consumer in the standalone rpc proxy
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
# disconnect the service's queue consumer while it's running a worker
eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn)
# we should receive the response from the first call
# the standalone RPC proxy will stop listening as soon as it receives
# a reply, so the duplicate response is discarded
arg = uuid.uuid4().hex
assert proxy.method(arg) == arg
# `method` will have been called twice with the same the `arg`, because
# rabbit will have redelivered the un-ack'd message from the first call
def method_called_twice():
assert method_called.call_args_list == [call(arg), call(arg)]
assert_stops_raising(method_called_twice)
connections = get_rabbit_connections(vhost, rabbit_manager)
assert queue_consumer_conn not in [conn['name'] for conn in connections]
rpc_proxy.stop()
def test_service_disconnect_with_active_rpc_worker(
container_factory, rabbit_manager, rabbit_config):
""" Break the connection between a service's queue consumer and rabbit
while the service has an active rpc worker (i.e. response required).
"""
container = container_factory(ExampleService, rabbit_config)
container.start()
# get the service's queue consumer connection while we know it's the
# only active connection
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
queue_consumer_conn = connections[0]['name']
# create a standalone RPC proxy towards the target service
rpc_proxy = ServiceRpcProxy('exampleservice', rabbit_config)
proxy = rpc_proxy.start()
# there should now be two connections:
# 1. the queue consumer from the target service
# 2. the queue consumer in the standalone rpc proxy
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
# disconnect the service's queue consumer while it's running a worker
eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn)
# we should receive the response from the first call
# the standalone RPC proxy will stop listening as soon as it receives
# a reply, so the duplicate response is discarded
arg = uuid.uuid4().hex
assert proxy.method(arg) == arg
# `method` will have been called twice with the same the `arg`, because
# rabbit will have redelivered the un-ack'd message from the first call
def method_called_twice():
assert method_called.call_args_list == [call(arg), call(arg)]
assert_stops_raising(method_called_twice)
connections = get_rabbit_connections(vhost, rabbit_manager)
assert queue_consumer_conn not in [conn['name'] for conn in connections]
rpc_proxy.stop()
def test_disconnect_with_pending_reply(container_factory, rabbit_manager,
rabbit_config):
example_container = container_factory(ExampleService, rabbit_config)
example_container.start()
vhost = rabbit_config['vhost']
# get exampleservice's queue consumer connection while we know it's the
# only active connection
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
container_connection = connections[0]
with ServiceRpcProxy('exampleservice', rabbit_config) as proxy:
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
proxy_connection = [
conn for conn in connections if conn != container_connection
][0]
def disconnect_once(self):
if hasattr(disconnect_once, 'called'):
return
disconnect_once.called = True
rabbit_manager.delete_connection(proxy_connection['name'])
with patch.object(ExampleService, 'callback', disconnect_once):
async = proxy.method.call_async('hello')
# if disconnecting while waiting for a reply, call fails
with pytest.raises(RpcConnectionError):
proxy.method('hello')
# the failure above also has to consider any other pending calls a
# failure, since the reply may have been sent while the queue was
# gone (deleted on disconnect, and not added until re-connect)
with pytest.raises(RpcConnectionError):
async .result()
# proxy should work again afterwards
assert proxy.method('hello') == 'hello'
def test_disconnect_with_pending_reply(
container_factory, rabbit_manager, rabbit_config):
example_container = container_factory(ExampleService, rabbit_config)
example_container.start()
vhost = rabbit_config['vhost']
# get exampleservice's queue consumer connection while we know it's the
# only active connection
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
container_connection = connections[0]
with ServiceRpcProxy('exampleservice', rabbit_config) as proxy:
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
proxy_connection = [
conn for conn in connections if conn != container_connection][0]
def disconnect_once(self):
if hasattr(disconnect_once, 'called'):
return
disconnect_once.called = True
rabbit_manager.delete_connection(proxy_connection['name'])
with patch.object(ExampleService, 'callback', disconnect_once):
async = proxy.method.call_async('hello')
# if disconnecting while waiting for a reply, call fails
with pytest.raises(RpcConnectionError):
proxy.method('hello')
# the failure above also has to consider any other pending calls a
# failure, since the reply may have been sent while the queue was
# gone (deleted on disconnect, and not added until re-connect)
with pytest.raises(RpcConnectionError):
async.result()
# proxy should work again afterwards
assert proxy.method('hello') == 'hello'
def test_get_rabbit_connections():
vhost = "vhost_name"
connections = [{
'vhost': vhost,
'key': 'value'
}, {
'vhost': 'unlikely_vhost_name',
'key': 'value'
}]
rabbit_manager = Mock()
rabbit_manager.get_connections.return_value = connections
vhost_conns = [connections[0]]
assert get_rabbit_connections(vhost, rabbit_manager) == vhost_conns
rabbit_manager.get_connections.return_value = None
assert get_rabbit_connections(vhost, rabbit_manager) == []
def test_proxy_disconnect_with_active_worker(container_factory, rabbit_manager,
rabbit_config):
""" Break the connection to rabbit while a service's queue consumer and
rabbit while the service has an in-flight rpc request (i.e. it is waiting
on a reply).
"""
# ExampleService is the target; ProxyService has the rpc_proxy;
proxy_container = container_factory(ProxyService, rabbit_config)
example_container = container_factory(ExampleService, rabbit_config)
proxy_container.start()
# get proxyservice's queue consumer connection while we know it's the
# only active connection
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
proxy_consumer_conn = connections[0]['name']
example_container.start()
# there should now be two connections:
# 1. the queue consumer from proxyservice
# 2. the queue consumer from exampleservice
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
# disconnect proxyservice's queue consumer while its request is in-flight
eventlet.spawn(disconnect_on_event, rabbit_manager, proxy_consumer_conn)
with entrypoint_hook(proxy_container, 'retry') as retry:
# if disconnecting while waiting for a reply, call fails
# fail, then success
assert retry('hello') == [
(RpcConnectionError, 'Disconnected while waiting for reply'),
'duplicate-call-result',
]
connections = get_rabbit_connections(vhost, rabbit_manager)
assert proxy_consumer_conn not in [conn['name'] for conn in connections]
def test_proxy_disconnect_with_active_worker(
container_factory, rabbit_manager, rabbit_config):
""" Break the connection to rabbit while a service's queue consumer and
rabbit while the service has an in-flight rpc request (i.e. it is waiting
on a reply).
"""
# ExampleService is the target; ProxyService has the rpc_proxy;
proxy_container = container_factory(ProxyService, rabbit_config)
example_container = container_factory(ExampleService, rabbit_config)
proxy_container.start()
# get proxyservice's queue consumer connection while we know it's the
# only active connection
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
proxy_consumer_conn = connections[0]['name']
example_container.start()
# there should now be two connections:
# 1. the queue consumer from proxyservice
# 2. the queue consumer from exampleservice
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
# disconnect proxyservice's queue consumer while its request is in-flight
eventlet.spawn(disconnect_on_event, rabbit_manager, proxy_consumer_conn)
with entrypoint_hook(proxy_container, 'retry') as retry:
# if disconnecting while waiting for a reply, call fails
# fail, then success
assert retry('hello') == [
(RpcConnectionError, 'Disconnected while waiting for reply'),
'duplicate-call-result',
]
connections = get_rabbit_connections(vhost, rabbit_manager)
assert proxy_consumer_conn not in [conn['name'] for conn in connections]
def check_connections():
""" Raise a runtime error if the test leaves any connections open.
Allow a few retries because the rabbit api is eventually consistent.
"""
connections = get_rabbit_connections(conf['vhost'], rabbit_manager)
open_connections = [
conn for conn in connections if conn['state'] != "closed"
]
if open_connections:
count = len(open_connections)
names = ", ".join(conn['name'] for conn in open_connections)
raise RuntimeError(
"{} rabbit connection(s) left open: {}".format(count, names))
def check_connections():
""" Raise a runtime error if the test leaves any connections open.
Allow a few retries because the rabbit api is eventually consistent.
"""
connections = get_rabbit_connections(conf['vhost'], rabbit_manager)
open_connections = [
conn for conn in connections if conn['state'] != "closed"
]
if open_connections:
count = len(open_connections)
names = ", ".join(conn['name'] for conn in open_connections)
raise RuntimeError(
"{} rabbit connection(s) left open: {}".format(count, names))
def rabbit_config(request, rabbit_manager):
import random
import string
from kombu import pools
from six.moves.urllib.parse import urlparse # pylint: disable=E0401
from nameko.testing.utils import get_rabbit_connections
amqp_uri = request.config.getoption('AMQP_URI')
uri = urlparse(amqp_uri)
username = uri.username
vhost = uri.path[1:]
use_random_vost = (vhost == ":random:")
if use_random_vost:
vhost = "test_{}".format(
"".join(random.choice(string.ascii_lowercase) for _ in range(10))
)
amqp_uri = "{}://{}/{}".format(uri.scheme, uri.netloc, vhost)
rabbit_manager.create_vhost(vhost)
rabbit_manager.set_vhost_permissions(vhost, username, '.*', '.*', '.*')
conf = {
'AMQP_URI': amqp_uri,
'username': username,
'vhost': vhost
}
yield conf
pools.reset() # close connections in pools
# raise a runtime error if the test leaves any connections lying around
try:
connections = get_rabbit_connections(conf['vhost'], rabbit_manager)
open_connections = [
conn for conn in connections if conn['state'] != "closed"
]
if open_connections:
count = len(open_connections)
names = ", ".join(conn['name'] for conn in open_connections)
raise RuntimeError(
"{} rabbit connection(s) left open: {}".format(count, names))
finally:
if use_random_vost:
rabbit_manager.delete_vhost(vhost)
def rabbit_config(request, rabbit_manager):
amqp_uri = request.config.getoption('AMQP_URI')
conf = get_rabbit_config(amqp_uri)
reset_rabbit_connections(conf['vhost'], rabbit_manager)
reset_rabbit_vhost(conf['vhost'], conf['username'], rabbit_manager)
yield conf
pools.reset() # close connections in pools
# raise a runtime error if the test leaves any connections lying around
connections = get_rabbit_connections(conf['vhost'], rabbit_manager)
if connections:
count = len(connections)
raise RuntimeError("{} rabbit connection(s) left open.".format(count))
def rabbit_config(request, rabbit_manager):
amqp_uri = request.config.getoption('AMQP_URI')
conf = get_rabbit_config(amqp_uri)
reset_rabbit_connections(conf['vhost'], rabbit_manager)
reset_rabbit_vhost(conf['vhost'], conf['username'], rabbit_manager)
yield conf
pools.reset() # close connections in pools
# raise a runtime error if the test leaves any connections lying around
connections = get_rabbit_connections(conf['vhost'], rabbit_manager)
if connections:
count = len(connections)
raise RuntimeError("{} rabbit connection(s) left open.".format(count))
def rabbit_config(request, rabbit_manager):
amqp_uri = request.config.getoption('AMQP_URI')
conf = {'AMQP_URI': amqp_uri}
uri = urlparse(amqp_uri)
vhost = uri.path[1:].replace('/', '%2F')
username = uri.username
conf['vhost'] = vhost
conf['username'] = username
reset_rabbit_connections(vhost, rabbit_manager)
reset_rabbit_vhost(vhost, username, rabbit_manager)
yield conf
pools.reset() # close connections in pools
# raise a runtime error if the test leaves any connections lying around
connections = get_rabbit_connections(vhost, rabbit_manager)
if connections:
count = len(connections)
raise RuntimeError("{} rabbit connection(s) left open.".format(count))
def rabbit_config(request, rabbit_manager):
amqp_uri = request.config.getoption('AMQP_URI')
conf = {'AMQP_URI': amqp_uri}
uri = urlparse(amqp_uri)
vhost = uri.path[1:].replace('/', '%2F')
username = uri.username
conf['vhost'] = vhost
conf['username'] = username
reset_rabbit_connections(vhost, rabbit_manager)
reset_rabbit_vhost(vhost, username, rabbit_manager)
yield conf
pools.reset() # close connections in pools
# raise a runtime error if the test leaves any connections lying around
connections = get_rabbit_connections(vhost, rabbit_manager)
if connections:
count = len(connections)
raise RuntimeError("{} rabbit connection(s) left open.".format(count))
def check_connections_closed():
connections = get_rabbit_connections(vhost, rabbit_manager)
if connections: # pragma: no cover
for connection in connections:
assert connection['vhost'] != vhost
def test_disconnect_with_pending_reply(container_factory, rabbit_manager,
rabbit_config):
block = Event()
class ExampleService(object):
name = "exampleservice"
def hook(self):
pass # pragma: no cover
@rpc
def method(self, arg):
self.hook()
block.wait()
return arg
container = container_factory(ExampleService, rabbit_config)
container.start()
vhost = rabbit_config['vhost']
# get exampleservice's queue consumer connection while we know it's the
# only active connection
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
container_connection = connections[0]
with ServiceRpcProxy('exampleservice', rabbit_config) as proxy:
# grab the proxy's connection too, the only other connection
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
proxy_connection = [
conn for conn in connections if conn != container_connection
][0]
counter = itertools.count(start=1)
class ConnectionStillOpen(Exception):
pass
@retry(for_exceptions=ConnectionStillOpen, delay=0.2)
def wait_for_connection_close(name):
connections = get_rabbit_connections(vhost, rabbit_manager)
for conn in connections:
if conn['name'] == name:
raise ConnectionStillOpen(name) # pragma: no cover
def cb(args, kwargs, res, exc_info):
# trigger a disconnection on the second call.
# release running workers once the connection has been closed
count = next(counter)
if count == 2:
rabbit_manager.delete_connection(proxy_connection['name'])
wait_for_connection_close(proxy_connection['name'])
block.send(True)
return True
# attach a callback to `hook` so we can close the connection
# while there are requests in-flight
with wait_for_call(ExampleService, 'hook', callback=cb):
# make an async call that runs for some time
async_call = proxy.method.call_async("hello")
# make another call that will trigger the disconnection;
# expect the blocking proxy to raise when the service reconnects
with pytest.raises(RpcConnectionError):
proxy.method("hello")
# also expect the running call to raise, since the reply may have
# been sent while the queue was gone (deleted on disconnect, and
# not added until re-connect)
with pytest.raises(RpcConnectionError):
async_call.result()
# proxy should work again afterwards
assert proxy.method("hello") == "hello"
def wait_for_connection_close(name):
connections = get_rabbit_connections(vhost, rabbit_manager)
for conn in connections:
if conn['name'] == name:
raise ConnectionStillOpen(name) # pragma: no cover
def check_connections_closed():
connections = get_rabbit_connections(vhost, rabbit_manager)
if connections: # pragma: no cover
for connection in connections:
assert connection['vhost'] != vhost
