def test_queue_basics(self) -> None:
queue_client = get_queue_client()
queue_client.publish("test_suite", 'test_event')
result = queue_client.drain_queue("test_suite")
self.assertEqual(len(result), 1)
self.assertEqual(result[0], b'test_event')
def setup_tornado_rabbitmq():
# type: () -> None
# When tornado is shut down, disconnect cleanly from rabbitmq
if settings.USING_RABBITMQ:
queue_client = get_queue_client()
atexit.register(lambda: queue_client.close())
tornado.autoreload.add_reload_hook(lambda: queue_client.close())
def test_queue_basics_json(self) -> None:
queue_json_publish("test_suite", {"event": "my_event"})
queue_client = get_queue_client()
result = queue_client.drain_queue("test_suite", json=True)
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['event'], 'my_event')
def test_queue_error_json(self) -> None:
queue_client = get_queue_client()
actual_publish = queue_client.publish
self.counter = 0
def throw_connection_error_once(self_obj: Any, *args: Any, **kwargs: Any) -> None:
self.counter += 1
if self.counter <= 1:
raise AMQPConnectionError("test")
actual_publish(*args, **kwargs)
with mock.patch(
"zerver.lib.queue.SimpleQueueClient.publish", throw_connection_error_once
), self.assertLogs("zulip.queue", level="WARN") as warn_logs:
queue_json_publish("test_suite", {"event": "my_event"})
self.assertEqual(
warn_logs.output,
["WARNING:zulip.queue:Failed to send to rabbitmq, trying to reconnect and send again"],
)
assert queue_client.channel
(_, _, message) = queue_client.channel.basic_get("test_suite")
assert message
result = orjson.loads(message)
self.assertEqual(result["event"], "my_event")
(_, _, message) = queue_client.channel.basic_get("test_suite")
assert not message
def test_queue_basics(self) -> None:
queue_client = get_queue_client()
queue_client.publish("test_suite", 'test_event')
result = queue_client.drain_queue("test_suite")
self.assertEqual(len(result), 1)
self.assertEqual(result[0], b'test_event')
def test_queue_basics_json(self) -> None:
queue_json_publish("test_suite", {"event": "my_event"})
queue_client = get_queue_client()
with queue_client.json_drain_queue("test_suite") as result:
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['event'], 'my_event')
def test_register_consumer_nack(self) -> None:
output = []
count = 0
queue_client = get_queue_client()
def collect(events: List[Dict[str, Any]]) -> None:
assert isinstance(queue_client, SimpleQueueClient)
assert len(events) == 1
queue_client.stop_consuming()
nonlocal count
count += 1
if count == 1:
raise Exception("Make me nack!")
output.append(events[0])
queue_json_publish("test_suite", {"event": "my_event"})
try:
queue_client.start_json_consumer("test_suite", collect)
except Exception:
queue_client.start_json_consumer("test_suite", collect)
# Confirm that we processed the event fully once
self.assertEqual(count, 2)
self.assert_length(output, 1)
self.assertEqual(output[0]["event"], "my_event")
def test_register_consumer_nack(self) -> None:
output = []
count = 0
queue_client = get_queue_client()
def collect(event: Dict[str, Any]) -> None:
queue_client.stop_consuming()
nonlocal count
count += 1
if count == 1:
raise Exception("Make me nack!")
output.append(event)
queue_client.register_json_consumer("test_suite", collect)
queue_json_publish("test_suite", {"event": "my_event"})
try:
queue_client.start_consuming()
except Exception:
queue_client.register_json_consumer("test_suite", collect)
queue_client.start_consuming()
# Confirm that we processed the event fully once
self.assertEqual(count, 2)
self.assertEqual(len(output), 1)
self.assertEqual(output[0]['event'], 'my_event')
def setup_tornado_rabbitmq():
# type: () -> None
# When tornado is shut down, disconnect cleanly from rabbitmq
if settings.USING_RABBITMQ:
queue_client = get_queue_client()
atexit.register(lambda: queue_client.close())
tornado.autoreload.add_reload_hook(lambda: queue_client.close()) # type: ignore # TODO: Fix missing tornado.autoreload stub
def test_register_consumer_nack(self) -> None:
output = []
count = 0
queue_client = get_queue_client()
def collect(event: Dict[str, Any]) -> None:
queue_client.stop_consuming()
nonlocal count
count += 1
if count == 1:
raise Exception("Make me nack!")
output.append(event)
queue_client.register_json_consumer("test_suite", collect)
queue_json_publish("test_suite", {"event": "my_event"})
try:
queue_client.start_consuming()
except Exception:
queue_client.register_json_consumer("test_suite", collect)
queue_client.start_consuming()
# Confirm that we processed the event fully once
self.assertEqual(count, 2)
self.assertEqual(len(output), 1)
self.assertEqual(output[0]['event'], 'my_event')
def inner_run():
# type: () -> None
from django.conf import settings
from django.utils import translation
translation.activate(settings.LANGUAGE_CODE)
print("Validating Django models.py...")
self.check(display_num_errors=True)
print("\nDjango version %s" % (django.get_version()))
print("Tornado server is running at http://%s:%s/" % (addr, port))
print("Quit the server with %s." % (quit_command, ))
if settings.USING_RABBITMQ:
queue_client = get_queue_client()
# Process notifications received via RabbitMQ
queue_client.register_json_consumer('notify_tornado',
process_notification)
queue_client.register_json_consumer('tornado_return',
respond_send_message)
try:
urls = (
r"/notify_tornado",
r"/json/events",
r"/api/v1/events",
)
# Application is an instance of Django's standard wsgi handler.
application = web.Application(
[(url, AsyncDjangoHandler)
for url in urls] + get_sockjs_router().urls,
debug=django.conf.settings.DEBUG,
# Disable Tornado's own request logging, since we have our own
log_function=lambda x: None)
# start tornado web server in single-threaded mode
http_server = httpserver.HTTPServer(
application,
xheaders=xheaders,
no_keep_alive=no_keep_alive)
http_server.listen(int(port), address=addr)
setup_event_queue()
add_client_gc_hook(missedmessage_hook)
setup_tornado_rabbitmq()
instance = ioloop.IOLoop.instance()
if django.conf.settings.DEBUG:
instance.set_blocking_log_threshold(5)
instance.handle_callback_exception = handle_callback_exception
instance.start()
except KeyboardInterrupt:
sys.exit(0)
def inner_run():
# type: () -> None
from django.conf import settings
from django.utils import translation
translation.activate(settings.LANGUAGE_CODE)
print("Validating Django models.py...")
self.validate(display_num_errors=True)
print("\nDjango version %s" % (django.get_version()))
print("Tornado server is running at http://%s:%s/" % (addr, port))
print("Quit the server with %s." % (quit_command,))
if settings.USING_RABBITMQ:
queue_client = get_queue_client()
# Process notifications received via RabbitMQ
queue_client.register_json_consumer('notify_tornado', process_notification)
queue_client.register_json_consumer('tornado_return', respond_send_message)
try:
urls = (r"/notify_tornado",
r"/json/events",
r"/api/v1/events",
)
# Application is an instance of Django's standard wsgi handler.
application = web.Application([(url, AsyncDjangoHandler) for url in urls]
+ get_sockjs_router().urls,
debug=django.conf.settings.DEBUG,
# Disable Tornado's own request logging, since we have our own
log_function=lambda x: None)
# start tornado web server in single-threaded mode
http_server = httpserver.HTTPServer(application,
xheaders=xheaders,
no_keep_alive=no_keep_alive)
http_server.listen(int(port), address=addr)
setup_event_queue()
add_client_gc_hook(missedmessage_hook)
setup_tornado_rabbitmq()
instance = ioloop.IOLoop.instance()
if django.conf.settings.DEBUG:
instance.set_blocking_log_threshold(5)
instance.handle_callback_exception=handle_callback_exception
instance.start()
except KeyboardInterrupt:
sys.exit(0)
def test_register_consumer(self) -> None:
output = []
queue_client = get_queue_client()
def collect(events: List[Dict[str, Any]]) -> None:
assert len(events) == 1
output.append(events[0])
queue_client.stop_consuming()
queue_json_publish("test_suite", {"event": "my_event"})
queue_client.start_json_consumer("test_suite", collect)
self.assertEqual(len(output), 1)
self.assertEqual(output[0]["event"], "my_event")
def test_register_consumer(self) -> None:
output = []
queue_client = get_queue_client()
def collect(event: Dict[str, Any]) -> None:
output.append(event)
queue_client.stop_consuming()
queue_client.register_json_consumer("test_suite", collect)
queue_json_publish("test_suite", {"event": "my_event"})
queue_client.start_consuming()
self.assertEqual(len(output), 1)
self.assertEqual(output[0]['event'], 'my_event')
def test_register_consumer(self) -> None:
output = []
queue_client = get_queue_client()
def collect(event: Dict[str, Any]) -> None:
output.append(event)
queue_client.stop_consuming()
queue_client.register_json_consumer("test_suite", collect)
queue_json_publish("test_suite", {"event": "my_event"})
queue_client.start_consuming()
self.assertEqual(len(output), 1)
self.assertEqual(output[0]['event'], 'my_event')
def inner_run() -> None:
from django.conf import settings
from django.utils import translation
translation.activate(settings.LANGUAGE_CODE)
# We pass display_num_errors=False, since Django will
# likely display similar output anyway.
self.check(display_num_errors=False)
print(f"Tornado server (re)started on port {port}")
if settings.USING_RABBITMQ:
queue_client = get_queue_client()
assert isinstance(queue_client, TornadoQueueClient)
# Process notifications received via RabbitMQ
queue_name = notify_tornado_queue_name(port)
queue_client.start_json_consumer(
queue_name, get_wrapped_process_notification(queue_name)
)
try:
# Application is an instance of Django's standard wsgi handler.
application = create_tornado_application()
if settings.AUTORELOAD:
zulip_autoreload_start()
# start tornado web server in single-threaded mode
http_server = httpserver.HTTPServer(
application, xheaders=xheaders, no_keep_alive=no_keep_alive
)
http_server.listen(port, address=addr)
from zerver.tornado.ioloop_logging import logging_data
logging_data["port"] = str(port)
setup_event_queue(port)
add_client_gc_hook(missedmessage_hook)
setup_tornado_rabbitmq()
instance = ioloop.IOLoop.instance()
if django.conf.settings.DEBUG:
instance.set_blocking_log_threshold(5)
instance.handle_callback_exception = handle_callback_exception
instance.start()
except KeyboardInterrupt:
sys.exit(0)
def inner_run() -> None:
from django.conf import settings
from django.utils import translation
translation.activate(settings.LANGUAGE_CODE)
print("Validating Django models.py...")
self.check(display_num_errors=True)
print("\nDjango version %s" % (django.get_version(), ))
print("Tornado server is running at http://%s:%s/" % (addr, port))
print("Quit the server with %s." % (quit_command, ))
if settings.USING_RABBITMQ:
queue_client = get_queue_client()
# Process notifications received via RabbitMQ
queue_client.register_json_consumer(
notify_tornado_queue_name(int(port)), process_notification)
queue_client.register_json_consumer(
tornado_return_queue_name(int(port)), respond_send_message)
try:
# Application is an instance of Django's standard wsgi handler.
application = create_tornado_application(int(port))
if settings.AUTORELOAD:
zulip_autoreload_start()
# start tornado web server in single-threaded mode
http_server = httpserver.HTTPServer(
application,
xheaders=xheaders,
no_keep_alive=no_keep_alive)
http_server.listen(int(port), address=addr)
setup_event_queue(int(port))
add_client_gc_hook(missedmessage_hook)
setup_tornado_rabbitmq()
from zerver.tornado.ioloop_logging import logging_data
logging_data['port'] = port
instance = ioloop.IOLoop.instance()
if django.conf.settings.DEBUG:
instance.set_blocking_log_threshold(5)
instance.handle_callback_exception = handle_callback_exception
instance.start()
except KeyboardInterrupt:
sys.exit(0)
def inner_run() -> None:
from django.conf import settings
from django.utils import translation
translation.activate(settings.LANGUAGE_CODE)
print("Validating Django models.py...")
self.check(display_num_errors=True)
print("\nDjango version %s" % (django.get_version()))
print("Tornado server is running at http://%s:%s/" % (addr, port))
print("Quit the server with %s." % (quit_command,))
if settings.USING_RABBITMQ:
queue_client = get_queue_client()
# Process notifications received via RabbitMQ
queue_client.register_json_consumer(notify_tornado_queue_name(int(port)),
process_notification)
queue_client.register_json_consumer(tornado_return_queue_name(int(port)),
respond_send_message)
try:
# Application is an instance of Django's standard wsgi handler.
application = create_tornado_application(int(port))
if settings.AUTORELOAD:
zulip_autoreload_start()
# start tornado web server in single-threaded mode
http_server = httpserver.HTTPServer(application,
xheaders=xheaders,
no_keep_alive=no_keep_alive)
http_server.listen(int(port), address=addr)
setup_event_queue(int(port))
add_client_gc_hook(missedmessage_hook)
setup_tornado_rabbitmq()
from zerver.tornado.ioloop_logging import logging_data
logging_data['port'] = port
instance = ioloop.IOLoop.instance()
if django.conf.settings.DEBUG:
instance.set_blocking_log_threshold(5)
instance.handle_callback_exception = handle_callback_exception
instance.start()
except KeyboardInterrupt:
sys.exit(0)
def test_queue_basics_json_error(self) -> None:
queue_json_publish("test_suite", {"event": "my_event"})
queue_client = get_queue_client()
raised = False
try:
with queue_client.json_drain_queue("test_suite") as result:
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['event'], 'my_event')
raise ValueError()
except ValueError:
raised = True
assert raised
# Still in the queue to be fetched
with queue_client.json_drain_queue("test_suite") as result:
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['event'], 'my_event')
def test_queue_error_json(self) -> None:
queue_client = get_queue_client()
actual_publish = queue_client.publish
self.counter = 0
def throw_connection_error_once(self_obj: Any, *args: Any,
**kwargs: Any) -> None:
self.counter += 1
if self.counter <= 1:
raise AMQPConnectionError("test")
actual_publish(*args, **kwargs)
with mock.patch("zerver.lib.queue.SimpleQueueClient.publish",
throw_connection_error_once):
queue_json_publish("test_suite", {"event": "my_event"})
result = queue_client.drain_queue("test_suite", json=True)
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['event'], 'my_event')
def test_queue_error_json(self) -> None:
queue_client = get_queue_client()
actual_publish = queue_client.publish
self.counter = 0
def throw_connection_error_once(self_obj: Any, *args: Any,
**kwargs: Any) -> None:
self.counter += 1
if self.counter <= 1:
raise AMQPConnectionError("test")
actual_publish(*args, **kwargs)
with mock.patch("zerver.lib.queue.SimpleQueueClient.publish",
throw_connection_error_once):
queue_json_publish("test_suite", {"event": "my_event"})
result = queue_client.drain_queue("test_suite", json=True)
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['event'], 'my_event')
def test_queue_error_json(self) -> None:
queue_client = get_queue_client()
actual_publish = queue_client.publish
self.counter = 0
def throw_connection_error_once(self_obj: Any, *args: Any,
**kwargs: Any) -> None:
self.counter += 1
if self.counter <= 1:
raise AMQPConnectionError("test")
actual_publish(*args, **kwargs)
with mock.patch("zerver.lib.queue.SimpleQueueClient.publish",
throw_connection_error_once), self.assertLogs('zulip.queue', level='WARN') as warn_logs:
queue_json_publish("test_suite", {"event": "my_event"})
self.assertEqual(warn_logs.output, [
'WARNING:zulip.queue:Failed to send to rabbitmq, trying to reconnect and send again'
])
result = queue_client.json_drain_queue("test_suite")
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['event'], 'my_event')
def setup_tornado_rabbitmq() -> None: # nocoverage
# When tornado is shut down, disconnect cleanly from rabbitmq
if settings.USING_RABBITMQ:
queue_client = get_queue_client()
atexit.register(lambda: queue_client.close())
autoreload.add_reload_hook(lambda: queue_client.close())
def test_queue_basics(self) -> None:
queue_client = get_queue_client()
queue_client.publish("test_suite", b"test_event\x00\xff")
with queue_client.drain_queue("test_suite") as result:
self.assertEqual(result, [b"test_event\x00\xff"])
def tearDown(self) -> None:
queue_client = get_queue_client()
with queue_client.drain_queue("test_suite"):
pass
super().tearDown()
def setUp(self) -> None:
queue_client = get_queue_client()
assert queue_client.channel
if "test_suite" in queue_client.queues:
queue_client.channel.queue_purge("test_suite")
super().setUp()
def setUp(self) -> None:
queue_client = get_queue_client()
assert queue_client.channel
queue_client.channel.queue_declare("test_suite", durable=True)
queue_client.channel.queue_purge("test_suite")
super().setUp()
def tearDown(self) -> None:
queue_client = get_queue_client()
assert queue_client.channel
queue_client.channel.queue_purge("test_suite")
super().tearDown()
def tearDown(self) -> None:
queue_client = get_queue_client()
queue_client.drain_queue("test_suite")
def tearDown(self) -> None:
queue_client = get_queue_client()
queue_client.drain_queue("test_suite")