def test_sensitive_arguments(self, container_factory, rabbit_config):
from examples.auth import JWT_SECRET
from examples.sensitive_arguments import Service
container = container_factory(Service, rabbit_config)
container.start()
with ServiceRpcProxy("service", rabbit_config) as proxy:
token = proxy.login("matt", "secret")
jwt.decode(token, key=JWT_SECRET, verify=True)
with pytest.raises(RemoteError) as exc:
proxy.login("matt", "incorrect")
assert exc.value.exc_type == "Unauthenticated"
def test_rpc_unknown_service(container_factory, rabbit_config):
container = container_factory(ExampleService, rabbit_config)
container.start()
with ServiceRpcProxy("exampleservice", rabbit_config) as proxy:
# success
assert proxy.task_a()
# failure
with pytest.raises(RemoteError) as exc_info:
proxy.call_unknown()
assert exc_info.value.exc_type == "UnknownService"
def test_async_timeout(
container_factory, rabbit_manager, rabbit_config
):
container = container_factory(FooService, rabbit_config)
container.start()
with ServiceRpcProxy('foobar', rabbit_config, timeout=.1) as proxy:
result = proxy.sleep.call_async(seconds=1)
with pytest.raises(RpcTimeout):
result.result()
result = proxy.sleep.call_async(seconds=.2)
eventlet.sleep(.2)
result.result()
def test_multiple_runners_coexist(runner_factory, rabbit_config,
rabbit_manager):
runner1 = runner_factory(rabbit_config, Service)
runner1.start()
runner2 = runner_factory(rabbit_config, Service)
runner2.start()
vhost = rabbit_config['vhost']
# verify there are two event queues with a single consumer each
def check_consumers():
evt_queues = [
queue for queue in rabbit_manager.get_queues(vhost)
if queue['name'].startswith('evt-srcservice-testevent')
]
assert len(evt_queues) == 2
for queue in evt_queues:
assert queue['consumers'] == 1
# rabbit's management API seems to lag
assert_stops_raising(check_consumers)
# test events (both services will receive if in "broadcast" mode)
event_data = "msg"
dispatch = event_dispatcher(rabbit_config)
dispatch('srcservice', "testevent", event_data)
with eventlet.Timeout(1):
while len(received) < 2:
eventlet.sleep()
assert received == [event_data, event_data]
# verify there are two consumers on the rpc queue
rpc_queue = rabbit_manager.get_queue(vhost, 'rpc-service')
assert rpc_queue['consumers'] == 2
# test rpc (only one service will respond)
del received[:]
arg = "msg"
with ServiceRpcProxy('service', rabbit_config) as proxy:
proxy.handle(arg)
with eventlet.Timeout(1):
while len(received) == 0:
eventlet.sleep()
assert received == [arg]
def test_multiple_runners_coexist(runner_factory, rabbit_config,
rabbit_manager, service_cls, tracker):
runner1 = runner_factory(rabbit_config, service_cls)
runner1.start()
runner2 = runner_factory(rabbit_config, service_cls)
runner2.start()
vhost = rabbit_config['vhost']
# verify there are two event queues with a single consumer each
def check_consumers():
evt_queues = [
queue for queue in rabbit_manager.get_queues(vhost)
if queue['name'].startswith('evt-srcservice-testevent')
]
assert len(evt_queues) == 2
for queue in evt_queues:
assert queue['consumers'] == 1
# rabbit's management API seems to lag
assert_stops_raising(check_consumers)
# test events (both services will receive if in "broadcast" mode)
event_data = "event"
dispatch = event_dispatcher(rabbit_config)
container1 = list(runner1.containers)[0]
container2 = list(runner2.containers)[0]
with entrypoint_waiter(container1, "handle"):
with entrypoint_waiter(container2, "handle"):
dispatch('srcservice', "testevent", event_data)
assert tracker.call_args_list == [call(event_data), call(event_data)]
# verify there are two consumers on the rpc queue
rpc_queue = rabbit_manager.get_queue(vhost, 'rpc-service')
assert rpc_queue['consumers'] == 2
# test rpc (only one service will respond)
arg = "arg"
with ServiceRpcProxy('service', rabbit_config) as proxy:
proxy.handle(arg)
assert tracker.call_args_list == [
call(event_data), call(event_data),
call(arg)
]
def test_expected_exceptions(self, container_factory, rabbit_config):
from examples.auth import JWT_SECRET
from examples.expected_exceptions import Service
container = container_factory(Service, rabbit_config)
container.start()
token = jwt.encode({"roles": []}, key=JWT_SECRET)
with ServiceRpcProxy(
"service", rabbit_config, context_data={"auth": token}
) as proxy:
with pytest.raises(RemoteError) as exc:
proxy.update(None)
assert exc.value.exc_type == 'Unauthorized'
admin_token = jwt.encode({"roles": ['admin']}, key=JWT_SECRET)
with ServiceRpcProxy(
"service", rabbit_config, context_data={"auth": admin_token}
) as proxy:
with pytest.raises(RemoteError) as exc:
proxy.update(None)
assert exc.value.exc_type == 'TypeError'
def test_unexpected_correlation_id(container_factory, rabbit_config):
container = container_factory(FooService, rabbit_config)
container.start()
with ServiceRpcProxy("foobar", rabbit_config) as proxy:
message = Message(channel=None, properties={
'reply_to': proxy.reply_listener.routing_key,
'correlation_id': 'invalid',
})
responder = Responder(container.config, message)
with patch('nameko.standalone.rpc._logger', autospec=True) as logger:
responder.send_response(None, None)
assert proxy.spam(ham='eggs') == 'eggs'
assert logger.debug.call_count == 1
def test_async_rpc(container_factory, rabbit_config):
container = container_factory(FooService, rabbit_config)
container.start()
with ServiceRpcProxy('foobar', rabbit_config) as foo:
rep1 = foo.spam.call_async(ham=1)
rep2 = foo.spam.call_async(ham=2)
rep3 = foo.spam.call_async(ham=3)
rep4 = foo.spam.call_async(ham=4)
rep5 = foo.spam.call_async(ham=5)
assert rep2.result() == 2
assert rep3.result() == 3
assert rep1.result() == 1
assert rep4.result() == 4
assert rep5.result() == 5
def test_rpc_proxy_over_ssl(self, container_factory, rabbit_ssl_config,
rabbit_config):
class Service(object):
name = "service"
@rpc
def echo(self, *args, **kwargs):
return args, kwargs
container = container_factory(Service, rabbit_config)
container.start()
with ServiceRpcProxy("service", rabbit_ssl_config) as proxy:
assert proxy.echo("a", "b", foo="bar") == [['a', 'b'], {
'foo': 'bar'
}]
def test_replace_dependencies_kwargs(container_factory, rabbit_config):
class Service(object):
name = "service"
foo_proxy = RpcProxy("foo_service")
bar_proxy = RpcProxy("bar_service")
baz_proxy = RpcProxy("baz_service")
@rpc
def method(self, arg):
self.foo_proxy.remote_method(arg)
class FakeDependency(object):
def __init__(self):
self.processed = []
def remote_method(self, arg):
self.processed.append(arg)
container = container_factory(Service, rabbit_config)
# customise a single dependency
fake_foo_proxy = FakeDependency()
replace_dependencies(container, foo_proxy=fake_foo_proxy)
assert 2 == len([
dependency for dependency in container.extensions
if isinstance(dependency, RpcProxy)
])
# customise multiple dependencies
res = replace_dependencies(container, bar_proxy=Mock(), baz_proxy=Mock())
assert list(res) == []
# verify that container.extensions doesn't include an RpcProxy anymore
assert all([
not isinstance(dependency, RpcProxy)
for dependency in container.extensions
])
container.start()
# verify that the fake dependency collected calls
msg = "msg"
with ServiceRpcProxy("service", rabbit_config) as service_proxy:
service_proxy.method(msg)
assert fake_foo_proxy.processed == [msg]
def test_service_disconnect_with_active_rpc_worker(container_factory,
rabbit_manager,
rabbit_config):
""" Break the connection between a service's queue consumer and rabbit
while the service has an active rpc worker (i.e. response required).
"""
container = container_factory(ExampleService, rabbit_config)
container.start()
# get the service's queue consumer connection while we know it's the
# only active connection
vhost = rabbit_config['vhost']
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
queue_consumer_conn = connections[0]['name']
# create a standalone RPC proxy towards the target service
rpc_proxy = ServiceRpcProxy('exampleservice', rabbit_config)
proxy = rpc_proxy.start()
# there should now be two connections:
# 1. the queue consumer from the target service
# 2. the queue consumer in the standalone rpc proxy
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
# disconnect the service's queue consumer while it's running a worker
eventlet.spawn(disconnect_on_event, rabbit_manager, queue_consumer_conn)
# we should receive the response from the first call
# the standalone RPC proxy will stop listening as soon as it receives
# a reply, so the duplicate response is discarded
arg = uuid.uuid4().hex
assert proxy.method(arg) == arg
# `method` will have been called twice with the same the `arg`, because
# rabbit will have redelivered the un-ack'd message from the first call
def method_called_twice():
assert method_called.call_args_list == [call(arg), call(arg)]
assert_stops_raising(method_called_twice)
connections = get_rabbit_connections(vhost, rabbit_manager)
assert queue_consumer_conn not in [conn['name'] for conn in connections]
rpc_proxy.stop()
def test_end_to_end(self, container_factory, rabbit_config):
class Service(object):
name = "service"
@rpc
def wait(self):
Event().wait()
container = container_factory(Service, rabbit_config)
container.start()
with ServiceRpcProxy("service", rabbit_config, timeout=.01) as proxy:
with pytest.raises(RpcTimeout):
proxy.wait()
# container won't stop gracefully with a running worker
container.kill()
def test_rpc_serialization(container_factory, rabbit_config,
sniffer_queue_factory, serializer):
config = rabbit_config
config[SERIALIZER_CONFIG_KEY] = serializer
container = container_factory(Service, config)
container.start()
get_messages = sniffer_queue_factory('nameko-rpc')
serialized = serialized_info[serializer]
with ServiceRpcProxy('service', rabbit_config) as proxy:
assert proxy.echo(test_data) == serialized['data']
assert entrypoint_called.call_args == call(serialized['data'])
msg = get_messages()[0]
assert msg['properties']['content_type'] == serialized['content_type']
def test_recover_from_keyboardinterrupt(container_factory, rabbit_manager,
rabbit_config):
container = container_factory(FooService, rabbit_config)
container.start() # create rpc queues
container.stop() # but make sure call doesn't complete
with ServiceRpcProxy('foobar', rabbit_config) as proxy:
with patch('kombu.connection.Connection.drain_events') as drain_events:
drain_events.side_effect = KeyboardInterrupt('killing from test')
with pytest.raises(KeyboardInterrupt):
proxy.spam(ham=0)
container = container_factory(FooService, rabbit_config)
container.start()
# proxy should still work
assert proxy.spam(ham=1) == 1
def test_replace_dependencies_args_and_kwargs(container_factory,
rabbit_config):
class Service(object):
name = "service"
foo_proxy = RpcProxy("foo_service")
bar_proxy = RpcProxy("bar_service")
baz_proxy = RpcProxy("baz_service")
@rpc
def method(self, arg):
self.foo_proxy.remote_method(arg)
self.bar_proxy.bar()
self.baz_proxy.baz()
class FakeDependency(object):
def __init__(self):
self.processed = []
def remote_method(self, arg):
self.processed.append(arg)
container = container_factory(Service, rabbit_config)
fake_foo_proxy = FakeDependency()
mock_bar_proxy, mock_baz_proxy = replace_dependencies(
container, 'bar_proxy', 'baz_proxy', foo_proxy=fake_foo_proxy)
# verify that container.extensions doesn't include an RpcProxy anymore
assert all([
not isinstance(dependency, RpcProxy)
for dependency in container.extensions
])
container.start()
# verify that the fake dependency collected calls
msg = "msg"
with ServiceRpcProxy("service", rabbit_config) as service_proxy:
service_proxy.method(msg)
assert fake_foo_proxy.processed == [msg]
assert mock_bar_proxy.bar.call_count == 1
assert mock_baz_proxy.baz.call_count == 1
def test_rpc_consumer_sharing(container_factory, rabbit_config,
rabbit_manager):
""" Verify that the RpcConsumer unregisters from the queueconsumer when
the first provider unregisters itself. Otherwise it keeps consuming
messages for the unregistered provider, raising MethodNotFound.
"""
container = container_factory(ExampleService, rabbit_config)
container.start()
task_a = get_extension(container, Rpc, method_name="task_a")
task_a_stop = task_a.stop
task_b = get_extension(container, Rpc, method_name="task_b")
task_b_stop = task_b.stop
task_a_stopped = Event()
def patched_task_a_stop():
task_a_stop() # stop immediately
task_a_stopped.send(True)
def patched_task_b_stop():
eventlet.sleep(2) # stop after 2 seconds
task_b_stop()
with patch.object(task_b, 'stop', patched_task_b_stop), \
patch.object(task_a, 'stop', patched_task_a_stop):
# stop the container and wait for task_a to stop
# task_b will still be in the process of stopping
eventlet.spawn(container.stop)
task_a_stopped.wait()
# try to call task_a.
# should timeout, rather than raising MethodNotFound
with ServiceRpcProxy("exampleservice", rabbit_config) as proxy:
with pytest.raises(eventlet.Timeout):
with eventlet.Timeout(1):
proxy.task_a()
# kill the container so we don't have to wait for task_b to stop
container.kill()
def test_disconnect_with_pending_reply(container_factory, rabbit_manager,
rabbit_config):
example_container = container_factory(ExampleService, rabbit_config)
example_container.start()
vhost = rabbit_config['vhost']
# get exampleservice's queue consumer connection while we know it's the
# only active connection
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
container_connection = connections[0]
with ServiceRpcProxy('exampleservice', rabbit_config) as proxy:
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
proxy_connection = [
conn for conn in connections if conn != container_connection
][0]
def disconnect_once(self):
if hasattr(disconnect_once, 'called'):
return
disconnect_once.called = True
rabbit_manager.delete_connection(proxy_connection['name'])
with patch.object(ExampleService, 'callback', disconnect_once):
async = proxy.method.call_async('hello')
# if disconnecting while waiting for a reply, call fails
with pytest.raises(RpcConnectionError):
proxy.method('hello')
# the failure above also has to consider any other pending calls a
# failure, since the reply may have been sent while the queue was
# gone (deleted on disconnect, and not added until re-connect)
with pytest.raises(RpcConnectionError):
async .result()
# proxy should work again afterwards
assert proxy.method('hello') == 'hello'
def test_events(self, container_factory, rabbit_config):
from examples.events import ServiceA, ServiceB
container_a = container_factory(ServiceA, rabbit_config)
container_b = container_factory(ServiceB, rabbit_config)
container_a.start()
container_b.start()
with ServiceRpcProxy('service_a', rabbit_config) as service_a_rpc:
with patch.object(ServiceB, 'handle_event') as handle_event:
with entrypoint_waiter(container_b, 'handle_event'):
service_a_rpc.dispatching_method("event payload")
assert handle_event.call_args_list == [call("event payload")]
# test without the patch to catch any errors in the handler method
with entrypoint_waiter(container_b, 'handle_event'):
service_a_rpc.dispatching_method("event payload")
def test_dependency_call_lifecycle_errors(container_factory, rabbit_config,
method_name):
container = container_factory(ExampleService, rabbit_config)
container.start()
dependency = get_extension(container, EventDispatcher)
with patch.object(dependency, method_name, autospec=True) as method:
err = "error in {}".format(method_name)
method.side_effect = Exception(err)
# use a standalone rpc proxy to call exampleservice.task()
with ServiceRpcProxy("exampleservice", rabbit_config) as proxy:
# proxy.task() will hang forever because it generates an error
proxy.task.call_async()
# verify that the error bubbles up to container.wait()
with pytest.raises(Exception) as exc_info:
container.wait()
assert str(exc_info.value) == err
def test_handle_result_error(container_factory, rabbit_config):
container = container_factory(ExampleService, rabbit_config)
container.start()
rpc_consumer = get_extension(container, RpcConsumer)
with patch.object(rpc_consumer, 'handle_result',
autospec=True) as handle_result:
err = "error in handle_result"
handle_result.side_effect = Exception(err)
# use a standalone rpc proxy to call exampleservice.task()
with ServiceRpcProxy("exampleservice", rabbit_config) as proxy:
# proxy.task() will hang forever because it generates an error
proxy.task.call_async()
with pytest.raises(Exception) as exc_info:
container.wait()
assert str(exc_info.value) == err
def test_worker_exception(
exception_cls, expected_level, container_factory, config
):
class Service(object):
name = "service"
sentry = SentryReporter()
@rpc(expected_exceptions=CustomException)
def broken(self):
raise exception_cls("Error!")
container = container_factory(Service, config)
container.start()
with entrypoint_waiter(container, 'broken') as result:
with ServiceRpcProxy('service', config) as rpc_proxy:
with pytest.raises(RemoteError):
rpc_proxy.broken()
with pytest.raises(exception_cls) as raised:
result.get()
sentry = get_extension(container, SentryReporter)
assert sentry.client.send.call_count == 1
# generate expected call args
expected_logger = "service.broken"
expected_message = "Unhandled exception in call {}: {} {!r}".format(
'service.broken.1', exception_cls.__name__, str(raised.value)
)
_, kwargs = sentry.client.send.call_args
assert kwargs['message'] == expected_message
assert kwargs['logger'] == expected_logger
assert kwargs['level'] == expected_level
assert kwargs['extra'] == ANY
assert kwargs['tags'] == ANY
assert kwargs['user'] == {}
def test_rpc_serialization(container_factory, rabbit_config):
container = container_factory(Service, rabbit_config)
container.start()
data = {
"hello": ("world",),
123: 456,
'abc': [7, 8, 9],
'foobar': 1.5,
}
expected = {
"hello": ["world", ],
'123': 456,
'abc': [7, 8, 9],
'foobar': 1.5,
}
with ServiceRpcProxy('service', rabbit_config) as proxy:
assert proxy.echo(data) == expected
assert entrypoint_called.call_args == call(expected)
def test_reply_queue_not_removed_while_in_use(rabbit_manager, rabbit_config,
container_factory):
def list_queues():
vhost = rabbit_config['vhost']
return [
queue['name'] for queue in rabbit_manager.get_queues(vhost=vhost)
]
container = container_factory(FooService, rabbit_config)
container.start()
# check proxy re-use
with ServiceRpcProxy('foobar', rabbit_config) as foo:
queues_before = list_queues()
# sleep for 2x TTL
assert foo.sleep(0.2) == 0.2
queues_between = list_queues()
assert foo.spam(ham='eggs') == 'eggs'
queues_after = list_queues()
assert queues_before == queues_between == queues_after
def test_recover_from_keyboardinterrupt(container_factory, rabbit_manager,
rabbit_config):
container = container_factory(FooService, rabbit_config)
container.start() # create rpc queues
container.stop() # but make sure call doesn't complete
with ServiceRpcProxy('foobar', rabbit_config) as proxy:
def call():
return proxy.spam(ham=0)
with patch('nameko.standalone.rpc.queue_iterator') as iterator:
iterator.side_effect = KeyboardInterrupt('killing from test')
with pytest.raises(KeyboardInterrupt):
proxy.spam(ham=0)
container = container_factory(FooService, rabbit_config)
container.start()
# proxy should still work
assert proxy.spam(ham=1) == 1
def test_handle_result(container_factory, rabbit_manager, rabbit_config):
""" Verify that `handle_result` can modify the return values of the worker,
such that other dependencies see the updated values.
"""
container = container_factory(ExampleService, rabbit_config)
container.start()
with ServiceRpcProxy('exampleservice', rabbit_config) as proxy:
assert proxy.echo("hello") == "hello"
with pytest.raises(RemoteError) as exc:
proxy.unserializable()
assert "is not JSON serializable" in str(exc.value)
wait_for_worker_idle(container)
# verify ResultCollector sees values returned from `handle_result`
assert worker_result_called == [
("hello", None),
("something went wrong", (TypeError, ANY, ANY)),
]
def test_restrict_entrypoints(container_factory, rabbit_config):
method_called = Mock()
class Service(object):
name = "service"
@rpc
@once("assert not seen")
def handler_one(self, arg):
method_called(arg)
@event_handler('srcservice', 'eventtype')
def handler_two(self, msg):
method_called(msg)
container = container_factory(Service, rabbit_config)
# disable the entrypoints on handler_one
restrict_entrypoints(container, "handler_two")
container.start()
# verify the rpc entrypoint on handler_one is disabled
with ServiceRpcProxy("service", rabbit_config) as service_proxy:
with pytest.raises(MethodNotFound) as exc_info:
service_proxy.handler_one("msg")
assert str(exc_info.value) == "handler_one"
# dispatch an event to handler_two
msg = "msg"
dispatch = event_dispatcher(rabbit_config)
with entrypoint_waiter(container, 'handler_two'):
dispatch('srcservice', 'eventtype', msg)
# method_called should have exactly one call, derived from the event
# handler and not from the disabled @once entrypoint
method_called.assert_called_once_with(msg)
def test_replace_dependencies(container_factory, rabbit_config):
class Service(object):
name = "service"
foo_proxy = RpcProxy("foo_service")
bar_proxy = RpcProxy("bar_service")
baz_proxy = RpcProxy("baz_service")
@rpc
def method(self, arg):
self.foo_proxy.remote_method(arg)
@rpc
def foo(self):
return "bar"
container = container_factory(Service, rabbit_config)
# replace a single dependency
foo_proxy = replace_dependencies(container, "foo_proxy")
# replace multiple dependencies
replacements = replace_dependencies(container, "bar_proxy", "baz_proxy")
assert len([x for x in replacements]) == 2
# verify that container.extensions doesn't include an RpcProxy anymore
assert all([
not isinstance(dependency, RpcProxy)
for dependency in container.extensions
])
container.start()
# verify that the mock dependency collects calls
msg = "msg"
with ServiceRpcProxy("service", rabbit_config) as service_proxy:
service_proxy.method(msg)
foo_proxy.remote_method.assert_called_once_with(msg)
def test_tags_defaults(self, container_factory, service_cls, config):
container = container_factory(service_cls, config)
container.start()
with ServiceRpcProxy('service', config) as rpc_proxy:
with pytest.raises(RemoteError):
rpc_proxy.broken()
sentry = get_extension(container, SentryReporter)
assert sentry.client.send.call_count == 1
expected_tags = {
'site': config['SENTRY']['CLIENT_CONFIG']['site'],
'call_id': 'service.broken.1',
'parent_call_id': 'standalone_rpc_proxy.call.0',
'service_name': 'service',
'method_name': 'broken'
}
_, kwargs = sentry.client.send.call_args
assert expected_tags == kwargs['tags']
def test_runner_catches_container_errors(runner_factory, rabbit_config):
runner = runner_factory(rabbit_config, ExampleService)
runner.start()
container = get_container(runner, ExampleService)
rpc_consumer = get_extension(container, RpcConsumer)
with patch.object(rpc_consumer, 'handle_result',
autospec=True) as handle_result:
exception = Exception("error")
handle_result.side_effect = exception
# use a standalone rpc proxy to call exampleservice.task()
with ServiceRpcProxy("exampleservice", rabbit_config) as proxy:
# proxy.task() will hang forever because it generates an error
# in the remote container (so never receives a response).
proxy.task.call_async()
# verify that the error bubbles up to runner.wait()
with pytest.raises(Exception) as exc_info:
runner.wait()
assert exc_info.value == exception
def test_prefetch_throughput(container_factory, rabbit_config):
"""Make sure even max_workers=1 can consumer faster than 1 msg/second
Regression test for https://github.com/nameko/nameko/issues/417
"""
class Service(object):
name = "service"
@rpc
def method(self):
pass
rabbit_config[MAX_WORKERS_CONFIG_KEY] = 1
container = container_factory(Service, rabbit_config)
container.start()
replies = []
with ServiceRpcProxy("service", rabbit_config) as proxy:
for _ in range(5):
replies.append(proxy.method.call_async())
with eventlet.Timeout(1):
[reply.result() for reply in replies]
