def test_callback_with_exception_multiple_calls(self):
class EchoException(Exception):
pass
class Echo(object):
def error(self):
raise exc
echo = Echo()
exc = EchoException("error!")
callback = Mock()
callback.side_effect = [False, True]
with wait_for_call(echo, 'error', callback):
with pytest.raises(EchoException):
echo.error()
with pytest.raises(EchoException):
echo.error()
assert callback.called
assert callback.call_args_list == [
call((), {}, None, (EchoException, exc, ANY)),
call((), {}, None, (EchoException, exc, ANY))
]
def test_callback_multiple_calls(self):
class Echo(object):
count = 0
def upper(self, arg):
self.count += 1
return "{}-{}".format(arg.upper(), self.count)
echo = Echo()
arg = "hello"
callback = Mock()
callback.side_effect = [False, True]
with wait_for_call(echo, 'upper', callback):
res1 = echo.upper(arg)
assert res1 == "HELLO-1"
res2 = echo.upper(arg)
assert res2 == "HELLO-2"
assert callback.called
assert callback.call_args_list == [
call((arg,), {}, res1, None),
call((arg,), {}, res2, None),
]
def test_wait_until_stops_raising(self, forever):
class ThresholdNotReached(Exception):
pass
class CounterWithThreshold(object):
def __init__(self, threshold):
self.value = 0
self.threshold = threshold
def count(self):
self.value += 1
if self.value < self.threshold:
raise ThresholdNotReached(self.threshold)
return self.value
threshold = 10
counter = CounterWithThreshold(threshold)
def count_forever():
while forever:
try:
counter.count()
except ThresholdNotReached:
pass
time.sleep(0)
def cb(args, kwargs, res, exc_info):
return exc_info is None
with wait_for_call(counter, 'count', callback=cb) as result:
Thread(target=count_forever).start()
assert result.get() == threshold
def test_callback_multiple_calls(self):
class Echo(object):
count = 0
def upper(self, arg):
self.count += 1
return "{}-{}".format(arg.upper(), self.count)
echo = Echo()
arg = "hello"
callback = Mock()
callback.side_effect = [False, True]
with wait_for_call(echo, 'upper', callback):
res1 = echo.upper(arg)
assert res1 == "HELLO-1"
res2 = echo.upper(arg)
assert res2 == "HELLO-2"
assert callback.called
assert callback.call_args_list == [
call((arg, ), {}, res1, None),
call((arg, ), {}, res2, None),
]
def test_callback_with_exception_multiple_calls(self):
class EchoException(Exception):
pass
class Echo(object):
def error(self):
raise exc
echo = Echo()
exc = EchoException("error!")
callback = Mock()
callback.side_effect = [False, True]
with wait_for_call(echo, 'error', callback):
with pytest.raises(EchoException):
echo.error()
with pytest.raises(EchoException):
echo.error()
assert callback.called
assert callback.call_args_list == [
call((), {}, None, (EchoException, exc, ANY)),
call((), {}, None, (EchoException, exc, ANY))
]
def test_wait_until_raises(self, forever):
class LimitExceeded(Exception):
pass
class CounterWithLimit(object):
def __init__(self, limit):
self.value = 0
self.limit = limit
def count(self):
self.value += 1
if self.value >= self.limit:
raise LimitExceeded(self.limit)
return self.value
limit = 10
counter = CounterWithLimit(limit)
def count_forever():
while forever:
counter.count()
time.sleep(0)
def cb(args, kwargs, res, exc_info):
return exc_info is not None
with wait_for_call(counter, 'count', callback=cb) as result:
Thread(target=count_forever).start()
with pytest.raises(LimitExceeded):
result.get()
def test_wait_until_raises(self, forever):
class LimitExceeded(Exception):
pass
class CounterWithLimit(object):
def __init__(self, limit):
self.value = 0
self.limit = limit
def count(self):
self.value += 1
if self.value >= self.limit:
raise LimitExceeded(self.limit)
return self.value
limit = 10
counter = CounterWithLimit(limit)
def count_forever():
while forever:
counter.count()
time.sleep(0)
def cb(args, kwargs, res, exc_info):
return exc_info is not None
with wait_for_call(counter, 'count', callback=cb) as result:
Thread(target=count_forever).start()
with pytest.raises(LimitExceeded):
result.get()
def test_run_with_rename_single_service(rabbit_config):
parser = setup_parser()
broker = rabbit_config['AMQP_URI']
ser_name = Service.name
args = parser.parse_args([
'run',
'--broker',
broker,
'--backdoor-port',
0,
'test.sample:Service',
'--rename',
'renamed_service',
])
with wait_for_call(ServiceRunner, 'start'):
gt = eventlet.spawn(main, args)
with ClusterRpcProxy(rabbit_config) as proxy:
proxy.renamed_service.ping()
pid = os.getpid()
os.kill(pid, signal.SIGTERM)
gt.wait()
Service.name = ser_name
def test_wait_until_stops_raising(self, forever):
class ThresholdNotReached(Exception):
pass
class CounterWithThreshold(object):
def __init__(self, threshold):
self.value = 0
self.threshold = threshold
def count(self):
self.value += 1
if self.value < self.threshold:
raise ThresholdNotReached(self.threshold)
return self.value
threshold = 10
counter = CounterWithThreshold(threshold)
def count_forever():
while forever:
try:
counter.count()
except ThresholdNotReached:
pass
time.sleep(0)
def cb(args, kwargs, res, exc_info):
return exc_info is None
with wait_for_call(counter, 'count', callback=cb) as result:
Thread(target=count_forever).start()
assert result.get() == threshold
def test_wait_until_called_with_argument(self, forever):
class CounterWithSet(object):
value = 0
def count(self):
self.value += 1
return self.value
def set(self, value):
self.value = value
return self.value
counter = CounterWithSet()
def increment_forever_via_set():
while forever:
counter.set(counter.value + 1)
time.sleep(0)
def cb(args, kwargs, res, exc_info):
return args == (10, )
with wait_for_call(counter, 'set', callback=cb) as result:
Thread(target=increment_forever_via_set).start()
assert result.get() == 10
def test_disconnect_and_successfully_reconnect(self, container_factory,
rabbit_manager,
rabbit_config,
toxic_rpc_proxy, toxiproxy):
block = Event()
class Service(object):
name = "service"
@rpc
def method(self, arg):
block.wait()
return arg
container = container_factory(Service, rabbit_config)
container.start()
# make an async call that will block,
# wait for the worker to have spawned
with wait_for_call(container, 'spawn_worker'):
res = toxic_rpc_proxy.service.method.call_async('msg1')
# disconnect toxiproxy
toxiproxy.disable()
# reconnect toxiproxy just before the consumer attempts to reconnect;
# (consumer.cancel is the only hook we have)
def reconnect(args, kwargs, res, exc_info):
block.send(True)
toxiproxy.enable()
return True
with wait_for_call(
toxic_rpc_proxy._reply_listener.queue_consumer.consumer,
'cancel',
callback=reconnect):
# rpc proxy should return an error for the request in flight.
with pytest.raises(RpcConnectionError):
res.result()
# proxy should work again after reconnection
assert toxic_rpc_proxy.service.method("msg3") == "msg3"
def test_patch_class(self):
class Echo(object):
def upper(self, arg):
return arg.upper()
echo = Echo()
arg = "hello"
with wait_for_call(Echo, 'upper'):
res = echo.upper(arg)
assert res == "HELLO"
def test_disconnect_and_successfully_reconnect(
self, container_factory, rabbit_manager, rabbit_config,
toxic_rpc_proxy, toxiproxy
):
block = Event()
class Service(object):
name = "service"
@rpc
def method(self, arg):
block.wait()
return arg
container = container_factory(Service, rabbit_config)
container.start()
# make an async call that will block,
# wait for the worker to have spawned
with wait_for_call(container, 'spawn_worker'):
res = toxic_rpc_proxy.service.method.call_async('msg1')
# disable toxiproxy to kill connections
toxiproxy.disable()
# re-enable toxiproxy just before the consumer attempts to reconnect;
# (consumer.cancel is the only hook we have)
def reconnect(args, kwargs, res, exc_info):
block.send(True)
toxiproxy.enable()
return True
with wait_for_call(
toxic_rpc_proxy._reply_listener.queue_consumer.consumer, 'cancel',
callback=reconnect
):
# rpc proxy should recover the message in flight
res.result() == "msg1"
# proxy should work again after reconnection
assert toxic_rpc_proxy.service.method("msg2") == "msg2"
def test_result(self):
class Echo(object):
def upper(self, arg):
return arg.upper()
echo = Echo()
arg = "hello"
with wait_for_call(echo, 'upper') as result:
res = echo.upper(arg)
assert result.get() == res
def test_patch_class(self):
class Echo(object):
def upper(self, arg):
return arg.upper()
echo = Echo()
arg = "hello"
with wait_for_call(Echo, 'upper'):
res = echo.upper(arg)
assert res == "HELLO"
def test_indirect(self):
class Echo(object):
def proxy(self, arg):
return self.upper(arg)
def upper(self, arg):
return arg.upper()
echo = Echo()
arg = "hello"
with wait_for_call(echo, 'upper'):
assert echo.proxy(arg) == "HELLO"
def test_wrapped_method_raises(self):
class EchoException(Exception):
pass
class Echo(object):
def error(self):
raise EchoException("error!")
echo = Echo()
with wait_for_call(echo, 'error'):
with pytest.raises(EchoException):
echo.error()
def test_result_is_none(self):
class Echo(object):
def nothing(self):
return None
echo = Echo()
with wait_for_call(echo, 'nothing') as result:
res = echo.nothing()
assert res is None
assert result.get() is None
assert result.has_result is True
def test_main_with_logging_config(rabbit_config, tmpdir):
config = """
AMQP_URI: {amqp_uri}
LOGGING:
version: 1
disable_existing_loggers: false
formatters:
simple:
format: "%(name)s - %(levelname)s - %(message)s"
handlers:
capture:
class: logging.FileHandler
level: INFO
formatter: simple
filename: {capture_file}
root:
level: INFO
handlers: [capture]
"""
capture_file = tmpdir.join('capture.log')
config_file = tmpdir.join('config.yaml')
config_file.write(
dedent(config.format(
capture_file=capture_file.strpath,
amqp_uri=rabbit_config['AMQP_URI']
))
)
parser = setup_parser()
args = parser.parse_args([
'run',
'--config',
config_file.strpath,
'test.sample',
])
# start runner and wait for it to come up
with wait_for_call(ServiceRunner, 'start'):
gt = eventlet.spawn(main, args)
with ClusterRpcProxy(rabbit_config) as proxy:
proxy.service.ping()
pid = os.getpid()
os.kill(pid, signal.SIGTERM)
gt.wait()
assert "test.sample - INFO - ping!" in capture_file.read()
def test_result_not_ready(self):
class Echo(object):
def upper(self, arg):
return arg.upper()
echo = Echo()
arg = "hello"
with wait_for_call(echo, 'upper') as result:
with pytest.raises(result.NotReady):
result.get()
res = echo.upper(arg)
assert result.get() == res
def test_result(self):
class Echo(object):
def upper(self, arg):
return arg.upper()
echo = Echo()
arg = "hello"
with wait_for_call(echo, 'upper') as result:
res = echo.upper(arg)
assert result.get() == res
def test_main_with_logging_config(rabbit_config, tmpdir):
config = """
AMQP_URI: {amqp_uri}
LOGGING:
version: 1
disable_existing_loggers: false
formatters:
simple:
format: "%(name)s - %(levelname)s - %(message)s"
handlers:
capture:
class: logging.FileHandler
level: INFO
formatter: simple
filename: {capture_file}
root:
level: INFO
handlers: [capture]
"""
capture_file = tmpdir.join('capture.log')
config_file = tmpdir.join('config.yaml')
config_file.write(
dedent(
config.format(capture_file=capture_file.strpath,
amqp_uri=rabbit_config['AMQP_URI'])))
parser = setup_parser()
args = parser.parse_args([
'run',
'--config',
config_file.strpath,
'test.sample',
])
# start runner and wait for it to come up
with wait_for_call(ServiceRunner, 'start'):
gt = eventlet.spawn(main, args)
with ClusterRpcProxy(rabbit_config) as proxy:
proxy.service.ping()
pid = os.getpid()
os.kill(pid, signal.SIGTERM)
gt.wait()
assert "test.sample - INFO - ping!" in capture_file.read()
def test_indirect(self):
class Echo(object):
def proxy(self, arg):
return self.upper(arg)
def upper(self, arg):
return arg.upper()
echo = Echo()
arg = "hello"
with wait_for_call(echo, 'upper'):
assert echo.proxy(arg) == "HELLO"
def test_result_is_none(self):
class Echo(object):
def nothing(self):
return None
echo = Echo()
with wait_for_call(echo, 'nothing') as result:
res = echo.nothing()
assert res is None
assert result.get() is None
assert result.has_result is True
def test_wrapped_method_raises(self):
class EchoException(Exception):
pass
class Echo(object):
def error(self):
raise EchoException("error!")
echo = Echo()
with wait_for_call(echo, 'error'):
with pytest.raises(EchoException):
echo.error()
def test_entrypoint_waiter_result_teardown_race(
container_factory, rabbit_config, counter
):
tracker = Mock()
class TrackingDependency(DependencyProvider):
def worker_result(self, worker_ctx, res, exc_info):
tracker.worker_result()
def worker_teardown(self, worker_ctx):
tracker.worker_teardown()
class Service(object):
name = "service"
tracker = TrackingDependency()
@event_handler('srcservice', 'eventtype')
def handle(self, msg):
tracker.handle(msg)
container = container_factory(Service, rabbit_config)
container.start()
def wait_for_two_calls(worker_ctx, res, exc_info):
return counter.count() > 1
dispatch = event_dispatcher(rabbit_config)
with entrypoint_waiter(container, 'handle', callback=wait_for_two_calls):
# dispatch the first message
# wait until teardown has happened
with wait_for_call(TrackingDependency, 'worker_teardown'):
dispatch('srcservice', 'eventtype', "msg")
assert tracker.worker_teardown.call_count == 1
assert tracker.worker_result.call_count == 1
assert tracker.handle.call_count == 1
# dispatch the second event
dispatch('srcservice', 'eventtype', "msg")
# we should wait for the second teardown to complete before exiting
# the entrypoint waiter
assert tracker.worker_teardown.call_count == 2
assert tracker.worker_result.call_count == 2
assert tracker.handle.call_count == 2
def test_entrypoint_waiter_result_teardown_race(
container_factory, rabbit_config, counter
):
tracker = Mock()
class TrackingDependency(DependencyProvider):
def worker_result(self, worker_ctx, res, exc_info):
tracker.worker_result()
def worker_teardown(self, worker_ctx):
tracker.worker_teardown()
class Service(object):
name = "service"
tracker = TrackingDependency()
@event_handler('srcservice', 'eventtype')
def handle(self, msg):
tracker.handle(msg)
container = container_factory(Service, rabbit_config)
container.start()
def wait_for_two_calls(worker_ctx, res, exc_info):
return counter.count() > 1
dispatch = event_dispatcher(rabbit_config)
with entrypoint_waiter(container, 'handle', callback=wait_for_two_calls):
# dispatch the first message
# wait until teardown has happened
with wait_for_call(TrackingDependency, 'worker_teardown'):
dispatch('srcservice', 'eventtype', "msg")
assert tracker.worker_teardown.call_count == 1
assert tracker.worker_result.call_count == 1
assert tracker.handle.call_count == 1
# dispatch the second event
dispatch('srcservice', 'eventtype', "msg")
# we should wait for the second teardown to complete before exiting
# the entrypoint waiter
assert tracker.worker_teardown.call_count == 2
assert tracker.worker_result.call_count == 2
assert tracker.handle.call_count == 2
def test_result_get_raises(self):
class EchoException(Exception):
pass
class Echo(object):
def error(self):
raise EchoException("error!")
echo = Echo()
with wait_for_call(echo, 'error') as result:
with pytest.raises(EchoException):
echo.error()
with pytest.raises(EchoException):
result.get()
def test_result_not_ready(self):
class Echo(object):
def upper(self, arg):
return arg.upper()
echo = Echo()
arg = "hello"
with wait_for_call(echo, 'upper') as result:
with pytest.raises(result.NotReady):
result.get()
res = echo.upper(arg)
assert result.get() == res
def test_callback(self):
class Echo(object):
def upper(self, arg):
return arg.upper()
echo = Echo()
arg = "hello"
callback = Mock()
callback.return_value = True
with wait_for_call(echo, 'upper', callback):
res = echo.upper(arg)
assert res == "HELLO"
assert callback.called
assert callback.call_args_list == [call((arg, ), {}, res, None)]
def test_target_as_mock(self):
class Klass(object):
def __init__(self):
self.attr = "value"
def method(self):
return self.attr.upper()
instance = Klass()
with patch.object(instance, 'attr') as patched_attr:
with wait_for_call(patched_attr, 'upper'):
instance.method()
assert patched_attr.upper.called
assert instance.attr.upper.called
def test_with_retry_policy(self, service_rpc, toxiproxy):
""" Verify we automatically recover from stale connections.
Publish confirms are required for this functionality. Without confirms
the later messages are silently lost and the test hangs waiting for a
response.
"""
assert service_rpc.echo(1) == 1
toxiproxy.disable()
def enable_after_retry(args, kwargs, res, exc_info):
toxiproxy.enable()
return True
# subsequent calls succeed (after reconnecting via retry policy)
with wait_for_call(Connection, 'connect', callback=enable_after_retry):
assert service_rpc.echo(2) == 2
def test_with_retry_policy(self, service_rpc, toxiproxy):
""" Verify we automatically recover from stale connections.
Publish confirms are required for this functionality. Without confirms
the later messages are silently lost and the test hangs waiting for a
response.
"""
assert service_rpc.echo(1) == 1
toxiproxy.disable()
def enable_after_retry(args, kwargs, res, exc_info):
toxiproxy.enable()
return True
# subsequent calls succeed (after reconnecting via retry policy)
with wait_for_call(Connection, 'connect', callback=enable_after_retry):
assert service_rpc.echo(2) == 2
def test_result_get_raises(self):
class EchoException(Exception):
pass
class Echo(object):
def error(self):
raise EchoException("error!")
echo = Echo()
with wait_for_call(echo, 'error') as result:
with pytest.raises(EchoException):
echo.error()
with pytest.raises(EchoException):
result.get()
def test_callback(self):
class Echo(object):
def upper(self, arg):
return arg.upper()
echo = Echo()
arg = "hello"
callback = Mock()
callback.return_value = True
with wait_for_call(echo, 'upper', callback):
res = echo.upper(arg)
assert res == "HELLO"
assert callback.called
assert callback.call_args_list == [call((arg,), {}, res, None)]
def test_target_as_mock(self):
class Klass(object):
def __init__(self):
self.attr = "value"
def method(self):
return self.attr.upper()
instance = Klass()
with patch.object(instance, 'attr') as patched_attr:
with wait_for_call(patched_attr, 'upper'):
instance.method()
assert patched_attr.upper.called
assert instance.attr.upper.called
def test_with_new_thread(self):
class Echo(object):
def proxy(self, arg):
Thread(target=self.upper, args=(arg, )).start()
def upper(self, arg):
return arg.upper()
echo = Echo()
arg = "hello"
callback = Mock()
callback.return_value = True
with wait_for_call(echo, 'upper', callback):
res = echo.proxy(arg)
assert res is None
assert callback.called
assert callback.call_args_list == [call((arg, ), {}, "HELLO", None)]
def test_disconnect_and_fail_to_reconnect(self, container_factory,
rabbit_manager, rabbit_config,
toxic_rpc_proxy, toxiproxy):
block = Event()
class Service(object):
name = "service"
@rpc
def method(self, arg):
block.wait()
return arg
container = container_factory(Service, rabbit_config)
container.start()
# make an async call that will block,
# wait for the worker to have spawned
with wait_for_call(container, 'spawn_worker'):
res = toxic_rpc_proxy.service.method.call_async('msg1')
try:
# disconnect
toxiproxy.disable()
# rpc proxy should return an error for the request in flight.
# it will also attempt to reconnect and throw on failure
# because toxiproxy is still disconnected
with pytest.raises(socket.error):
with pytest.raises(RpcConnectionError):
res.result()
finally:
# reconnect toxiproxy
block.send(True)
toxiproxy.enable()
# proxy will not work afterwards because the queueconsumer connection
# was not recovered on the second attempt
with pytest.raises(RuntimeError):
toxic_rpc_proxy.service.method("msg3")
def test_failed_delivery(self, container, publish_message, exchange, queue,
rabbit_config):
backoff_publisher = get_extension(container, BackoffPublisher)
make_queue = backoff_publisher.make_queue
# patch make_queue so that the return value does not have
# a matching binding; this forces an unroutable messsage
with patch.object(backoff_publisher, 'make_queue') as patched:
patched.return_value = make_queue(999999)
# patch get_producer so we can wait until publish is called
# multiple times, demonstrating the retry
with patch('nameko_amqp_retry.backoff.get_producer') as patched:
# create a replacement producer that we can hook into
# and make our patched get_producer return that
amqp_uri = rabbit_config['AMQP_URI']
with get_producer(amqp_uri) as replacement_producer:
@contextmanager
def producer_context(*a, **k):
yield replacement_producer
patched.side_effect = producer_context
# fire entrypoint and wait for retry of the backoff publish
counter = itertools.count()
with wait_for_call(
replacement_producer,
'publish',
callback=lambda *a, **k: next(counter) == 2):
publish_message(exchange,
"",
routing_key=queue.routing_key)
# when the retry also fails,
# the container is killed so that the request is requeued
with pytest.raises(UndeliverableMessage):
container.wait()
def test_callback_with_exception(self):
class EchoException(Exception):
pass
class Echo(object):
def error(self):
raise exc
echo = Echo()
exc = EchoException("error!")
callback = Mock()
callback.return_value = True
with wait_for_call(echo, 'error', callback):
with pytest.raises(EchoException):
echo.error()
assert callback.called
assert callback.call_args_list == [
call((), {}, None, (EchoException, exc, ANY))
]
def test_with_new_thread(self):
class Echo(object):
def proxy(self, arg):
Thread(target=self.upper, args=(arg,)).start()
def upper(self, arg):
return arg.upper()
echo = Echo()
arg = "hello"
callback = Mock()
callback.return_value = True
with wait_for_call(echo, 'upper', callback):
res = echo.proxy(arg)
assert res is None
assert callback.called
assert callback.call_args_list == [call((arg,), {}, "HELLO", None)]
def test_wait_for_specific_result(self, forever):
class Counter(object):
value = 0
def count(self):
self.value += 1
return self.value
counter = Counter()
def count_forever():
while forever:
counter.count()
time.sleep(0)
def cb(args, kwargs, res, exc_info):
return res == 10
with wait_for_call(counter, 'count', callback=cb) as result:
Thread(target=count_forever).start()
assert result.get() == 10
def test_wait_until_called_with_argument(self, forever):
class CounterWithSet(object):
value = 0
def set(self, value):
self.value = value
return self.value
counter = CounterWithSet()
def increment_forever_via_set():
while forever:
counter.set(counter.value + 1)
time.sleep(0)
def cb(args, kwargs, res, exc_info):
return args == (10,)
with wait_for_call(counter, 'set', callback=cb) as result:
Thread(target=increment_forever_via_set).start()
assert result.get() == 10
def test_wait_for_specific_result(self, forever):
class Counter(object):
value = 0
def count(self):
self.value += 1
return self.value
counter = Counter()
def count_forever():
while forever:
counter.count()
time.sleep(0)
def cb(args, kwargs, res, exc_info):
return res == 10
with wait_for_call(counter, 'count', callback=cb) as result:
Thread(target=count_forever).start()
assert result.get() == 10
def test_disconnect_and_fail_to_reconnect(
self, container_factory, rabbit_manager, rabbit_config,
toxic_rpc_proxy, toxiproxy
):
block = Event()
class Service(object):
name = "service"
@rpc
def method(self, arg):
block.wait()
return arg
container = container_factory(Service, rabbit_config)
container.start()
# make an async call that will block,
# wait for the worker to have spawned
with wait_for_call(container, 'spawn_worker'):
res = toxic_rpc_proxy.service.method.call_async('msg1')
# disable toxiproxy to kill connections
with toxiproxy.disabled():
# toxiproxy remains disabled when the proxy attempts to reconnect,
# so we should return an error for the request in flight
with pytest.raises(socket.error):
res.result()
# unblock worker
block.send(True)
# proxy will not work afterwards because the queueconsumer connection
# was not recovered on the second attempt
with pytest.raises(RuntimeError):
toxic_rpc_proxy.service.method("msg2")
def test_disconnect_and_fail_to_reconnect(
self, container_factory, rabbit_manager, rabbit_config,
toxic_rpc_proxy, toxiproxy
):
block = Event()
class Service(object):
name = "service"
@rpc
def method(self, arg):
block.wait()
return arg
container = container_factory(Service, rabbit_config)
container.start()
# make an async call that will block,
# wait for the worker to have spawned
with wait_for_call(container, 'spawn_worker'):
res = toxic_rpc_proxy.service.method.call_async('msg1')
# disable toxiproxy to kill connections
with toxiproxy.disabled():
# toxiproxy remains disabled when the proxy attempts to reconnect,
# so we should return an error for the request in flight
with pytest.raises(socket.error):
res.result()
# unblock worker
block.send(True)
# proxy will not work afterwards because the queueconsumer connection
# was not recovered on the second attempt
with pytest.raises(RuntimeError):
toxic_rpc_proxy.service.method("msg2")
def test_close_socket(self, server, load_stubs, spec_dir, grpc_port, protobufs):
""" Regression test for https://github.com/nameko/nameko-grpc/issues/39
"""
stubs = load_stubs("example")
client = Client(
"//localhost:{}".format(grpc_port),
stubs.exampleStub,
"none",
"high",
False,
)
proxy = client.start()
container = server
grpc_server = get_extension(container, GrpcServer)
connection_ref = grpc_server.channel.conn_pool.connections.queue[0]
connection = connection_ref()
response = proxy.unary_unary(protobufs.ExampleRequest(value="A"))
assert response.message == "A"
with wait_for_call(connection.sock, "close"):
client.stop()
def test_run(rabbit_config):
parser = setup_parser()
broker = rabbit_config['AMQP_URI']
args = parser.parse_args([
'run',
'--broker',
broker,
'--backdoor-port',
0,
'test.sample:Service',
])
# start runner and wait for it to come up
with wait_for_call(ServiceRunner, 'start'):
gt = eventlet.spawn(main, args)
# make sure service launches ok
with ClusterRpcProxy(rabbit_config) as proxy:
proxy.service.ping()
# stop service
pid = os.getpid()
os.kill(pid, signal.SIGTERM)
gt.wait()
def test_run(rabbit_config):
parser = setup_parser()
broker = rabbit_config['AMQP_URI']
args = parser.parse_args([
'run',
'--broker',
broker,
'--backdoor-port',
0,
'test.sample:Service',
])
# start runner and wait for it to come up
with wait_for_call(ServiceRunner, 'start'):
gt = eventlet.spawn(main, args)
# make sure service launches ok
with ClusterRpcProxy(rabbit_config) as proxy:
proxy.service.ping()
# stop service
pid = os.getpid()
os.kill(pid, signal.SIGTERM)
gt.wait()
def test_callback_with_exception(self):
class EchoException(Exception):
pass
class Echo(object):
def error(self):
raise exc
echo = Echo()
exc = EchoException("error!")
callback = Mock()
callback.return_value = True
with wait_for_call(echo, 'error', callback):
with pytest.raises(EchoException):
echo.error()
assert callback.called
assert callback.call_args_list == [
call((), {}, None, (EchoException, exc, ANY))
]
def entrypoint_waiter(container, method_name, timeout=30, callback=None):
""" Context manager that waits until an entrypoint has fired, and
the generated worker has exited and been torn down.
It yields a :class:`nameko.testing.waiting.WaitResult` object that can be
used to get the result returned (exception raised) by the entrypoint
after the waiter has exited.
:Parameters:
container : ServiceContainer
The container hosting the service owning the entrypoint
method_name : str
The name of the entrypoint decorated method on the service class
timeout : int
Maximum seconds to wait
callback : callable
Function to conditionally control whether the entrypoint_waiter
should exit for a particular invocation
The `timeout` argument specifies the maximum number of seconds the
`entrypoint_waiter` should wait before exiting. It can be disabled by
passing `None`. The default is 30 seconds.
Optionally allows a `callback` to be provided which is invoked whenever
the entrypoint fires. If provided, the callback must return `True`
for the `entrypoint_waiter` to exit. The signature for the callback
function is::
def callback(worker_ctx, result, exc_info):
pass
Where there parameters are as follows:
worker_ctx (WorkerContext): WorkerContext of the entrypoint call.
result (object): The return value of the entrypoint.
exc_info (tuple): Tuple as returned by `sys.exc_info` if the
entrypoint raised an exception, otherwise `None`.
**Usage**
::
class Service(object):
name = "service"
@event_handler('srcservice', 'eventtype')
def handle_event(self, msg):
return msg
container = ServiceContainer(Service, config)
container.start()
# basic
with entrypoint_waiter(container, 'handle_event'):
... # action that dispatches event
# giving access to the result
with entrypoint_waiter(container, 'handle_event') as result:
... # action that dispatches event
res = result.get()
# with custom timeout
with entrypoint_waiter(container, 'handle_event', timeout=5):
... # action that dispatches event
# with callback that waits until entrypoint stops raising
def callback(worker_ctx, result, exc_info):
if exc_info is None:
return True
with entrypoint_waiter(container, 'handle_event', callback=callback):
... # action that dispatches event
"""
if not get_extension(container, Entrypoint, method_name=method_name):
raise RuntimeError("{} has no entrypoint `{}`".format(
container.service_name, method_name))
class Result(WaitResult):
worker_ctx = None
def send(self, worker_ctx, result, exc_info):
self.worker_ctx = worker_ctx
super(Result, self).send(result, exc_info)
waiter_callback = callback
waiter_result = Result()
def on_worker_result(worker_ctx, result, exc_info):
complete = False
if worker_ctx.entrypoint.method_name == method_name:
if not callable(waiter_callback):
complete = True
else:
complete = waiter_callback(worker_ctx, result, exc_info)
if complete:
waiter_result.send(worker_ctx, result, exc_info)
return complete
def on_worker_teardown(worker_ctx):
if waiter_result.worker_ctx is worker_ctx:
return True
return False
exc = entrypoint_waiter.Timeout("Timeout on {}.{} after {} seconds".format(
container.service_name, method_name, timeout))
with eventlet.Timeout(timeout, exception=exc):
with wait_for_call(
container, '_worker_teardown',
lambda args, kwargs, res, exc: on_worker_teardown(*args)):
with wait_for_call(
container, '_worker_result',
lambda args, kwargs, res, exc: on_worker_result(*args)):
yield waiter_result
def entrypoint_waiter(container, method_name, timeout=30, callback=None):
""" Context manager that waits until an entrypoint has fired, and
the generated worker has exited and been torn down.
It yields a :class:`nameko.testing.waiting.WaitResult` object that can be
used to get the result returned (exception raised) by the entrypoint
after the waiter has exited.
:Parameters:
container : ServiceContainer
The container hosting the service owning the entrypoint
method_name : str
The name of the entrypoint decorated method on the service class
timeout : int
Maximum seconds to wait
callback : callable
Function to conditionally control whether the entrypoint_waiter
should exit for a particular invocation
The `timeout` argument specifies the maximum number of seconds the
`entrypoint_waiter` should wait before exiting. It can be disabled by
passing `None`. The default is 30 seconds.
Optionally allows a `callback` to be provided which is invoked whenever
the entrypoint fires. If provided, the callback must return `True`
for the `entrypoint_waiter` to exit. The signature for the callback
function is::
def callback(worker_ctx, result, exc_info):
pass
Where there parameters are as follows:
worker_ctx (WorkerContext): WorkerContext of the entrypoint call.
result (object): The return value of the entrypoint.
exc_info (tuple): Tuple as returned by `sys.exc_info` if the
entrypoint raised an exception, otherwise `None`.
**Usage**
::
class Service(object):
name = "service"
@event_handler('srcservice', 'eventtype')
def handle_event(self, msg):
return msg
container = ServiceContainer(Service, config)
container.start()
# basic
with entrypoint_waiter(container, 'handle_event'):
... # action that dispatches event
# giving access to the result
with entrypoint_waiter(container, 'handle_event') as result:
... # action that dispatches event
res = result.get()
# with custom timeout
with entrypoint_waiter(container, 'handle_event', timeout=5):
... # action that dispatches event
# with callback that waits until entrypoint stops raising
def callback(worker_ctx, result, exc_info):
if exc_info is None:
return True
with entrypoint_waiter(container, 'handle_event', callback=callback):
... # action that dispatches event
"""
if not get_extension(container, Entrypoint, method_name=method_name):
raise RuntimeError("{} has no entrypoint `{}`".format(
container.service_name, method_name))
class Result(WaitResult):
worker_ctx = None
def send(self, worker_ctx, result, exc_info):
self.worker_ctx = worker_ctx
super(Result, self).send(result, exc_info)
waiter_callback = callback
waiter_result = Result()
def on_worker_result(worker_ctx, result, exc_info):
complete = False
if worker_ctx.entrypoint.method_name == method_name:
if not callable(waiter_callback):
complete = True
else:
complete = waiter_callback(worker_ctx, result, exc_info)
if complete:
waiter_result.send(worker_ctx, result, exc_info)
return complete
def on_worker_teardown(worker_ctx):
if waiter_result.worker_ctx is worker_ctx:
return True
return False
exc = entrypoint_waiter.Timeout(
"Timeout on {}.{} after {} seconds".format(
container.service_name, method_name, timeout)
)
with eventlet.Timeout(timeout, exception=exc):
with wait_for_call(
container, '_worker_teardown',
lambda args, kwargs, res, exc: on_worker_teardown(*args)
):
with wait_for_call(
container, '_worker_result',
lambda args, kwargs, res, exc: on_worker_result(*args)
):
yield waiter_result
def test_disconnect_with_pending_reply(container_factory, rabbit_manager,
rabbit_config):
block = Event()
class ExampleService(object):
name = "exampleservice"
def hook(self):
pass # pragma: no cover
@rpc
def method(self, arg):
self.hook()
block.wait()
return arg
container = container_factory(ExampleService, rabbit_config)
container.start()
vhost = rabbit_config['vhost']
# get exampleservice's queue consumer connection while we know it's the
# only active connection
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 1
container_connection = connections[0]
with ServiceRpcProxy('exampleservice', rabbit_config) as proxy:
# grab the proxy's connection too, the only other connection
connections = get_rabbit_connections(vhost, rabbit_manager)
assert len(connections) == 2
proxy_connection = [
conn for conn in connections if conn != container_connection
][0]
counter = itertools.count(start=1)
class ConnectionStillOpen(Exception):
pass
@retry(for_exceptions=ConnectionStillOpen, delay=0.2)
def wait_for_connection_close(name):
connections = get_rabbit_connections(vhost, rabbit_manager)
for conn in connections:
if conn['name'] == name:
raise ConnectionStillOpen(name) # pragma: no cover
def cb(args, kwargs, res, exc_info):
# trigger a disconnection on the second call.
# release running workers once the connection has been closed
count = next(counter)
if count == 2:
rabbit_manager.delete_connection(proxy_connection['name'])
wait_for_connection_close(proxy_connection['name'])
block.send(True)
return True
# attach a callback to `hook` so we can close the connection
# while there are requests in-flight
with wait_for_call(ExampleService, 'hook', callback=cb):
# make an async call that runs for some time
async_call = proxy.method.call_async("hello")
# make another call that will trigger the disconnection;
# expect the blocking proxy to raise when the service reconnects
with pytest.raises(RpcConnectionError):
proxy.method("hello")
# also expect the running call to raise, since the reply may have
# been sent while the queue was gone (deleted on disconnect, and
# not added until re-connect)
with pytest.raises(RpcConnectionError):
async_call.result()
# proxy should work again afterwards
assert proxy.method("hello") == "hello"
