def test_link_preview_non_html_data(self) -> None:
user = self.example_user('hamlet')
self.login_user(user)
url = 'http://test.org/audio.mp3'
with mock_queue_publish(
'zerver.lib.actions.queue_json_publish') as patched:
msg_id = self.send_stream_message(user,
"Scotland",
topic_name="foo",
content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
headers = {'content-type': 'application/octet-stream'}
mocked_response = mock.Mock(
side_effect=self.create_mock_response(url, headers=headers))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response), self.assertLogs(
level='INFO') as info_logs:
FetchLinksEmbedData().consume(event)
cached_data = link_embed_data_from_cache(url)
self.assertTrue(
'INFO:root:Time spent on get_link_embed_data for http://test.org/audio.mp3: '
in info_logs.output[0])
self.assertIsNone(cached_data)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertEqual(('<p><a href="http://test.org/audio.mp3">'
'http://test.org/audio.mp3</a></p>'),
msg.rendered_content)
def check_will_notify(self, **kwargs: Any) -> Tuple[str, str]:
hamlet = self.example_user("hamlet")
kwargs["user_profile_id"] = hamlet.id
kwargs["message_id"] = 32
# Fill up the parameters which weren't sent with defaults.
kwargs = self.get_maybe_enqueue_notifications_parameters(**kwargs)
email_notice = None
mobile_notice = None
with mock_queue_publish("zerver.tornado.event_queue.queue_json_publish"
) as mock_queue_json_publish:
notified = maybe_enqueue_notifications(**kwargs)
for entry in mock_queue_json_publish.call_args_list:
args = entry[0]
if args[0] == "missedmessage_mobile_notifications":
mobile_notice = args[1]
if args[0] == "missedmessage_emails":
email_notice = args[1]
# Now verify the return value matches the queue actions
if email_notice:
self.assertTrue(notified["email_notified"])
else:
self.assertFalse(notified.get("email_notified", False))
if mobile_notice:
self.assertTrue(notified["push_notified"])
else:
self.assertFalse(notified.get("push_notified", False))
return email_notice, mobile_notice
def test_youtube_url_title_replaces_url(self) -> None:
url = "https://www.youtube.com/watch?v=eSJTXC7Ixgg"
with mock_queue_publish("zerver.lib.actions.queue_json_publish"):
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [url],
"message_realm_id": msg.sender.realm_id,
"message_content": url,
}
mocked_data = {"title": "Clearer Code at Scale - Static Types at Zulip and Dropbox"}
self.create_mock_response(url)
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
with mock.patch(
"zerver.lib.markdown.link_preview.link_embed_data_from_cache",
lambda *args, **kwargs: mocked_data,
):
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for https://www.youtube.com/watch?v=eSJTXC7Ixgg:"
in info_logs.output[0]
)
msg.refresh_from_db()
expected_content = f"""<p><a href="https://www.youtube.com/watch?v=eSJTXC7Ixgg">YouTube - Clearer Code at Scale - Static Types at Zulip and Dropbox</a></p>\n<div class="youtube-video message_inline_image"><a data-id="eSJTXC7Ixgg" href="https://www.youtube.com/watch?v=eSJTXC7Ixgg"><img src="{get_camo_url("https://i.ytimg.com/vi/eSJTXC7Ixgg/default.jpg")}"></a></div>"""
self.assertEqual(expected_content, msg.rendered_content)
def test_edit_message_history(self) -> None:
user = self.example_user('hamlet')
self.login_user(user)
msg_id = self.send_stream_message(user,
"Scotland",
topic_name="editing",
content="original")
url = 'http://test.org/'
mocked_response = mock.Mock(side_effect=self.create_mock_response(url))
with mock_queue_publish(
'zerver.views.message_edit.queue_json_publish') as patched:
result = self.client_patch("/json/messages/" + str(msg_id), {
'message_id': msg_id,
'content': url,
})
self.assert_json_success(result)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response), self.assertLogs(
level='INFO') as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
'INFO:root:Time spent on get_link_embed_data for http://test.org/: '
in info_logs.output[0])
embedded_link = f'<a href="{url}" title="The Rock">The Rock</a>'
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertIn(embedded_link, msg.rendered_content)
def test_link_preview_non_html_data(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/audio.mp3"
with mock_queue_publish("zerver.lib.actions.queue_json_publish") as patched:
msg_id = self.send_stream_message(user, "Denmark", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
content_type = "application/octet-stream"
self.create_mock_response(url, content_type=content_type)
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
cached_data = link_embed_data_from_cache(url)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/audio.mp3: "
in info_logs.output[0]
)
self.assertIsNone(cached_data)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertEqual(
('<p><a href="http://test.org/audio.mp3">' "http://test.org/audio.mp3</a></p>"),
msg.rendered_content,
)
def test_invalid_url(self) -> None:
url = "http://test.org/"
error_url = "http://test.org/x"
with mock_queue_publish("zerver.lib.actions.queue_json_publish"):
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=error_url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [error_url],
"message_realm_id": msg.sender.realm_id,
"message_content": error_url,
}
self.create_mock_response(error_url, status=404)
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/x: "
in info_logs.output[0]
)
cached_data = link_embed_data_from_cache(error_url)
# FIXME: Should we really cache this, especially without cache invalidation?
self.assertIsNone(cached_data)
msg.refresh_from_db()
self.assertEqual(
'<p><a href="http://test.org/x">http://test.org/x</a></p>', msg.rendered_content
)
self.assertTrue(responses.assert_call_count(url, 0))
def test_link_preview_open_graph_image_missing_content(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/foo.html"
with mock_queue_publish("zerver.lib.actions.queue_json_publish") as patched:
msg_id = self.send_stream_message(user, "Denmark", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
# HTML without the og:image metadata
html = "\n".join(
line if "og:image" not in line else '<meta property="og:image"/>'
for line in self.open_graph_html.splitlines()
)
self.create_mock_response(url, body=html)
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
cached_data = link_embed_data_from_cache(url)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/foo.html: "
in info_logs.output[0]
)
self.assertIn("title", cached_data)
self.assertNotIn("image", cached_data)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertEqual(
('<p><a href="http://test.org/foo.html">' "http://test.org/foo.html</a></p>"),
msg.rendered_content,
)
def test_edit_message_history(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
msg_id = self.send_stream_message(user, "Denmark", topic_name="editing", content="original")
url = "http://test.org/"
self.create_mock_response(url)
with mock_queue_publish("zerver.lib.actions.queue_json_publish") as patched:
result = self.client_patch(
"/json/messages/" + str(msg_id),
{
"content": url,
},
)
self.assert_json_success(result)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0]
)
embedded_link = f'<a href="{url}" title="The Rock">The Rock</a>'
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertIn(embedded_link, msg.rendered_content)
def test_link_preview_css_escaping_image(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/"
with mock_queue_publish("zerver.lib.actions.queue_json_publish") as patched:
msg_id = self.send_stream_message(user, "Denmark", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
# Swap the URL out for one with characters that need CSS escaping
html = re.sub(r"rock\.jpg", "rock).jpg", self.open_graph_html)
self.create_mock_response(url, body=html)
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0]
)
msg = Message.objects.select_related("sender").get(id=msg_id)
with_preview = (
'<p><a href="http://test.org/">http://test.org/</a></p>\n<div class="message_embed"><a class="message_embed_image" href="http://test.org/" style="background-image: url('
+ "http\\:\\/\\/ia\\.media-imdb\\.com\\/images\\/rock\\)\\.jpg"
+ ')"></a><div class="data-container"><div class="message_embed_title"><a href="http://test.org/" title="The Rock">The Rock</a></div><div class="message_embed_description">Description text</div></div></div>'
)
self.assertEqual(
with_preview,
msg.rendered_content,
)
def test_message_deleted(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/"
with mock_queue_publish("zerver.actions.message_send.queue_json_publish") as patched:
msg_id = self.send_stream_message(user, "Denmark", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
msg = Message.objects.select_related("sender").get(id=msg_id)
do_delete_messages(msg.sender.realm, [msg])
# We do still fetch the URL, as we don't want to incur the
# cost of locking the row while we do the HTTP fetches.
self.create_mock_response(url)
with self.settings(TEST_SUITE=False):
with self.assertLogs(level="INFO") as info_logs:
# Run the queue processor. This will simulate the event for original_url being
# processed after the message has been deleted.
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0]
)
def test_email_sending_worker_retries(self) -> None:
"""Tests the retry_send_email_failures decorator to make sure it
retries sending the email 3 times and then gives up."""
fake_client = self.FakeClient()
data = {
"template_prefix": "zerver/emails/confirm_new_email",
"to_emails": [self.example_email("hamlet")],
"from_name": "Zulip Account Security",
"from_address": FromAddress.NOREPLY,
"context": {},
}
fake_client.enqueue("email_senders", data)
def fake_publish(queue_name: str, event: Dict[str, Any],
processor: Optional[Callable[[Any], None]]) -> None:
fake_client.enqueue(queue_name, event)
with simulated_queue_client(lambda: fake_client):
worker = queue_processors.EmailSendingWorker()
worker.setup()
with patch("zerver.lib.send_email.build_email",
side_effect=EmailNotDeliveredException
), mock_queue_publish(
"zerver.lib.queue.queue_json_publish",
side_effect=fake_publish), self.assertLogs(
level="ERROR") as m:
worker.start()
self.assertIn(
"failed due to exception EmailNotDeliveredException",
m.output[0])
self.assertEqual(data["failed_tries"], 1 + MAX_REQUEST_RETRIES)
def test_signups_worker_retries(self) -> None:
"""Tests the retry logic of signups queue."""
fake_client = self.FakeClient()
user_id = self.example_user('hamlet').id
data = {'user_id': user_id, 'id': 'test_missed'}
fake_client.queue.append(('signups', data))
def fake_publish(queue_name: str, event: Dict[str, Any], processor: Optional[Callable[[Any], None]]) -> None:
fake_client.queue.append((queue_name, event))
fake_response = MagicMock()
fake_response.status_code = 400
fake_response.content = orjson.dumps({'title': ''})
with simulated_queue_client(lambda: fake_client):
worker = queue_processors.SignupWorker()
worker.setup()
with patch('zerver.worker.queue_processors.requests.post',
return_value=fake_response), \
mock_queue_publish('zerver.lib.queue.queue_json_publish',
side_effect=fake_publish), \
patch('logging.info'), \
self.settings(MAILCHIMP_API_KEY='one-two',
PRODUCTION=True,
ZULIP_FRIENDS_LIST_ID='id'):
worker.start()
self.assertEqual(data['failed_tries'], 1 + MAX_REQUEST_RETRIES)
def test_email_sending_worker_retries(self) -> None:
"""Tests the retry_send_email_failures decorator to make sure it
retries sending the email 3 times and then gives up."""
fake_client = self.FakeClient()
data = {
'template_prefix': 'zerver/emails/confirm_new_email',
'to_emails': [self.example_email("hamlet")],
'from_name': 'Zulip Account Security',
'from_address': FromAddress.NOREPLY,
'context': {},
}
fake_client.queue.append(('email_senders', data))
def fake_publish(queue_name: str,
event: Dict[str, Any],
processor: Optional[Callable[[Any], None]]) -> None:
fake_client.queue.append((queue_name, event))
with simulated_queue_client(lambda: fake_client):
worker = queue_processors.EmailSendingWorker()
worker.setup()
with patch('zerver.lib.send_email.build_email',
side_effect=smtplib.SMTPServerDisconnected), \
mock_queue_publish('zerver.lib.queue.queue_json_publish',
side_effect=fake_publish), \
patch('logging.exception'):
worker.start()
self.assertEqual(data['failed_tries'], 1 + MAX_REQUEST_RETRIES)
def test_link_preview_no_content_type_header(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/"
with mock_queue_publish("zerver.lib.actions.queue_json_publish") as patched:
msg_id = self.send_stream_message(user, "Denmark", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
self.create_mock_response(url)
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
data = link_embed_data_from_cache(url)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0]
)
self.assertIn("title", data)
self.assertIn("image", data)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertIn(data["title"], msg.rendered_content)
self.assertIn(re.sub(r"([^\w-])", r"\\\1", data["image"]), msg.rendered_content)
def test_maybe_enqueue_notifications(self) -> None:
# We've already tested the "when to send notifications" logic as part of the
# notification_data module.
# This test is for verifying whether `maybe_enqueue_notifications` returns the
# `already_notified` data correctly.
params = self.get_maybe_enqueue_notifications_parameters(
message_id=1, user_id=1, acting_user_id=2)
with mock_queue_publish("zerver.tornado.event_queue.queue_json_publish"
) as mock_queue_json_publish:
notified = maybe_enqueue_notifications(**params)
mock_queue_json_publish.assert_not_called()
with mock_queue_publish("zerver.tornado.event_queue.queue_json_publish"
) as mock_queue_json_publish:
params["private_message"] = True
notified = maybe_enqueue_notifications(**params)
self.assertTrue(mock_queue_json_publish.call_count, 2)
queues_pushed = [
entry[0][0] for entry in mock_queue_json_publish.call_args_list
]
self.assertIn("missedmessage_mobile_notifications", queues_pushed)
self.assertIn("missedmessage_emails", queues_pushed)
self.assertTrue(notified["email_notified"])
self.assertTrue(notified["push_notified"])
with mock_queue_publish("zerver.tornado.event_queue.queue_json_publish"
) as mock_queue_json_publish:
params = self.get_maybe_enqueue_notifications_parameters(
message_id=1,
acting_user_id=2,
user_id=3,
private_message=False,
flags=["mentioned"],
mentioned=True,
mentioned_user_group_id=33,
)
notified = maybe_enqueue_notifications(**params)
self.assertTrue(mock_queue_json_publish.call_count, 2)
push_notice = mock_queue_json_publish.call_args_list[0][0][1]
self.assertEqual(push_notice["mentioned_user_group_id"], 33)
email_notice = mock_queue_json_publish.call_args_list[1][0][1]
self.assertEqual(email_notice["mentioned_user_group_id"], 33)
def test_push_notifications_worker(self) -> None:
"""
The push notifications system has its own comprehensive test suite,
so we can limit ourselves to simple unit testing the queue processor,
without going deeper into the system - by mocking the handle_push_notification
functions to immediately produce the effect we want, to test its handling by the queue
processor.
"""
fake_client = self.FakeClient()
def fake_publish(queue_name: str,
event: Dict[str, Any],
processor: Callable[[Any], None]) -> None:
fake_client.queue.append((queue_name, event))
def generate_new_message_notification() -> Dict[str, Any]:
return build_offline_notification(1, 1)
def generate_remove_notification() -> Dict[str, Any]:
return {
"type": "remove",
"user_profile_id": 1,
"message_ids": [1],
}
with simulated_queue_client(lambda: fake_client):
worker = queue_processors.PushNotificationsWorker()
worker.setup()
with patch('zerver.worker.queue_processors.handle_push_notification') as mock_handle_new, \
patch('zerver.worker.queue_processors.handle_remove_push_notification') as mock_handle_remove, \
patch('zerver.worker.queue_processors.initialize_push_notifications'):
event_new = generate_new_message_notification()
event_remove = generate_remove_notification()
fake_client.queue.append(('missedmessage_mobile_notifications', event_new))
fake_client.queue.append(('missedmessage_mobile_notifications', event_remove))
worker.start()
mock_handle_new.assert_called_once_with(event_new['user_profile_id'], event_new)
mock_handle_remove.assert_called_once_with(event_remove['user_profile_id'],
event_remove['message_ids'])
with patch('zerver.worker.queue_processors.handle_push_notification',
side_effect=PushNotificationBouncerRetryLaterError("test")) as mock_handle_new, \
patch('zerver.worker.queue_processors.handle_remove_push_notification',
side_effect=PushNotificationBouncerRetryLaterError("test")) as mock_handle_remove, \
patch('zerver.worker.queue_processors.initialize_push_notifications'):
event_new = generate_new_message_notification()
event_remove = generate_remove_notification()
fake_client.queue.append(('missedmessage_mobile_notifications', event_new))
fake_client.queue.append(('missedmessage_mobile_notifications', event_remove))
with mock_queue_publish('zerver.lib.queue.queue_json_publish', side_effect=fake_publish), \
self.assertLogs('zerver.worker.queue_processors', 'WARNING') as warn_logs:
worker.start()
self.assertEqual(mock_handle_new.call_count, 1 + MAX_REQUEST_RETRIES)
self.assertEqual(mock_handle_remove.call_count, 1 + MAX_REQUEST_RETRIES)
self.assertEqual(warn_logs.output, [
'WARNING:zerver.worker.queue_processors:Maximum retries exceeded for trigger:1 event:push_notification',
] * 2)
def test_inline_relative_url_embed_preview(self) -> None:
# Relative URLs should not be sent for URL preview.
with mock_queue_publish("zerver.lib.actions.queue_json_publish") as patched:
self.send_personal_message(
self.example_user("prospero"),
self.example_user("cordelia"),
content="http://zulip.testserver/api/",
)
patched.assert_not_called()
def _send_message_with_test_org_url(self,
sender: UserProfile,
queue_should_run: bool = True,
relative_url: bool = False) -> Message:
url = "http://test.org/"
with mock_queue_publish(
"zerver.lib.actions.queue_json_publish") as patched:
msg_id = self.send_personal_message(
sender,
self.example_user("cordelia"),
content=url,
)
if queue_should_run:
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
else:
patched.assert_not_called()
# If we nothing was put in the queue, we don't need to
# run the queue processor or any of the following code
return Message.objects.select_related("sender").get(id=msg_id)
# Verify the initial message doesn't have the embedded links rendered
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertNotIn(f'<a href="{url}" title="The Rock">The Rock</a>',
msg.rendered_content)
# Mock the network request result so the test can be fast without Internet
mocked_response = mock.Mock(side_effect=self.create_mock_response(
url, relative_url=relative_url))
# Run the queue processor to potentially rerender things
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch("requests.get", mocked_response), self.assertLogs(
level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0])
msg = Message.objects.select_related("sender").get(id=msg_id)
return msg
def test_valid_content_type_error_get_data(self) -> None:
url = "http://test.org/"
with mock_queue_publish(
"zerver.actions.message_send.queue_json_publish"):
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [url],
"message_realm_id": msg.sender.realm_id,
"message_content": url,
}
self.create_mock_response(url, body=ConnectionError())
with mock.patch(
"zerver.lib.url_preview.preview.get_oembed_data",
side_effect=lambda *args, **kwargs: None,
):
with mock.patch(
"zerver.lib.url_preview.preview.valid_content_type",
side_effect=lambda k: True):
with self.settings(TEST_SUITE=False):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0])
# This did not get cached -- hence the lack of [0] on the cache_get
cached_data = cache_get(preview_url_cache_key(url))
self.assertIsNone(cached_data)
msg.refresh_from_db()
self.assertEqual(
'<p><a href="http://test.org/">http://test.org/</a></p>',
msg.rendered_content)
def test_safe_oembed_html_url(self) -> None:
url = "http://test.org/"
with mock_queue_publish("zerver.lib.actions.queue_json_publish"):
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [url],
"message_realm_id": msg.sender.realm_id,
"message_content": url,
}
mocked_data = {
"html": f'<iframe src="{url}"></iframe>',
"oembed": True,
"type": "video",
"image": f"{url}/image.png",
}
mocked_response = mock.Mock(side_effect=self.create_mock_response(url))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch("requests.get", mocked_response), self.assertLogs(
level="INFO"
) as info_logs:
with mock.patch(
"zerver.lib.url_preview.preview.get_oembed_data",
lambda *args, **kwargs: mocked_data,
):
FetchLinksEmbedData().consume(event)
data = link_embed_data_from_cache(url)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0]
)
self.assertEqual(data, mocked_data)
msg.refresh_from_db()
self.assertIn('a data-id="{}"'.format(escape(mocked_data["html"])), msg.rendered_content)
def test_valid_content_type_error_get_data(self) -> None:
url = "http://test.org/"
with mock_queue_publish("zerver.lib.actions.queue_json_publish"):
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [url],
"message_realm_id": msg.sender.realm_id,
"message_content": url,
}
with mock.patch(
"zerver.lib.url_preview.preview.get_oembed_data",
side_effect=lambda *args, **kwargs: None,
):
with mock.patch(
"zerver.lib.url_preview.preview.valid_content_type",
side_effect=lambda k: True):
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch(
"requests.get",
mock.Mock(side_effect=ConnectionError(
))), self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0])
with self.assertRaises(NotFoundInCache):
link_embed_data_from_cache(url)
msg.refresh_from_db()
self.assertEqual(
'<p><a href="http://test.org/">http://test.org/</a></p>',
msg.rendered_content)
def test_safe_oembed_html_url(self) -> None:
url = 'http://test.org/'
with mock_queue_publish('zerver.lib.actions.queue_json_publish'):
msg_id = self.send_personal_message(
self.example_user('hamlet'),
self.example_user('cordelia'),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
'message_id': msg_id,
'urls': [url],
'message_realm_id': msg.sender.realm_id,
'message_content': url
}
mocked_data = {
'html': f'<iframe src="{url}"></iframe>',
'oembed': True,
'type': 'video',
'image': f'{url}/image.png'
}
mocked_response = mock.Mock(side_effect=self.create_mock_response(url))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response), self.assertLogs(
level='INFO') as info_logs:
with mock.patch(
'zerver.lib.url_preview.preview.get_oembed_data',
lambda *args, **kwargs: mocked_data):
FetchLinksEmbedData().consume(event)
data = link_embed_data_from_cache(url)
self.assertTrue(
'INFO:root:Time spent on get_link_embed_data for http://test.org/: '
in info_logs.output[0])
self.assertEqual(data, mocked_data)
msg.refresh_from_db()
self.assertIn('a data-id="{}"'.format(escape(mocked_data['html'])),
msg.rendered_content)
def check_will_notify(self, *args: Any, **kwargs: Any) -> Tuple[str, str]:
email_notice = None
mobile_notice = None
with mock_queue_publish("zerver.tornado.event_queue.queue_json_publish") as mock_queue_json_publish:
notified = maybe_enqueue_notifications(*args, **kwargs)
for entry in mock_queue_json_publish.call_args_list:
args = entry[0]
if args[0] == "missedmessage_mobile_notifications":
mobile_notice = args[1]
if args[0] == "missedmessage_emails":
email_notice = args[1]
# Now verify the return value matches the queue actions
if email_notice:
self.assertTrue(notified['email_notified'])
else:
self.assertFalse(notified.get('email_notified', False))
if mobile_notice:
self.assertTrue(notified['push_notified'])
else:
self.assertFalse(notified.get('push_notified', False))
return email_notice, mobile_notice
def test_safe_oembed_html_url(self) -> None:
url = "http://test.org/"
with mock_queue_publish(
"zerver.actions.message_send.queue_json_publish"):
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [url],
"message_realm_id": msg.sender.realm_id,
"message_content": url,
}
mocked_data = UrlOEmbedData(
html=f'<iframe src="{url}"></iframe>',
type="video",
image=f"{url}/image.png",
)
self.create_mock_response(url)
with self.settings(TEST_SUITE=False):
with self.assertLogs(level="INFO") as info_logs:
with mock.patch(
"zerver.lib.url_preview.preview.get_oembed_data",
lambda *args, **kwargs: mocked_data,
):
FetchLinksEmbedData().consume(event)
cached_data = cache_get(preview_url_cache_key(url))[0]
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0])
self.assertEqual(cached_data, mocked_data)
msg.refresh_from_db()
self.assertIn(f'a data-id="{escape(mocked_data.html)}"',
msg.rendered_content)
def test_link_preview_open_graph_image_missing_content(self) -> None:
user = self.example_user('hamlet')
self.login_user(user)
url = 'http://test.org/foo.html'
with mock_queue_publish(
'zerver.lib.actions.queue_json_publish') as patched:
msg_id = self.send_stream_message(user,
"Scotland",
topic_name="foo",
content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
# HTML without the og:image metadata
html = '\n'.join(
line if 'og:image' not in line else '<meta property="og:image"/>'
for line in self.open_graph_html.splitlines())
mocked_response = mock.Mock(
side_effect=self.create_mock_response(url, html=html))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response), self.assertLogs(
level='INFO') as info_logs:
FetchLinksEmbedData().consume(event)
cached_data = link_embed_data_from_cache(url)
self.assertTrue(
'INFO:root:Time spent on get_link_embed_data for http://test.org/foo.html: '
in info_logs.output[0])
self.assertIn('title', cached_data)
self.assertNotIn('image', cached_data)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertEqual(('<p><a href="http://test.org/foo.html">'
'http://test.org/foo.html</a></p>'),
msg.rendered_content)
def test_message_update_race_condition(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
original_url = "http://test.org/"
edited_url = "http://edited.org/"
with mock_queue_publish(
"zerver.lib.actions.queue_json_publish") as patched:
msg_id = self.send_stream_message(user,
"Scotland",
topic_name="foo",
content=original_url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
def wrapped_queue_json_publish(*args: Any, **kwargs: Any) -> None:
# Mock the network request result so the test can be fast without Internet
mocked_response_original = mock.Mock(
side_effect=self.create_mock_response(original_url))
mocked_response_edited = mock.Mock(
side_effect=self.create_mock_response(edited_url))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch("requests.get",
mocked_response_original), self.assertLogs(
level="INFO") as info_logs:
# Run the queue processor. This will simulate the event for original_url being
# processed after the message has been edited.
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0])
msg = Message.objects.select_related("sender").get(id=msg_id)
# The content of the message has changed since the event for original_url has been created,
# it should not be rendered. Another, up-to-date event will have been sent (edited_url).
self.assertNotIn(
f'<a href="{original_url}" title="The Rock">The Rock</a>',
msg.rendered_content)
mocked_response_edited.assert_not_called()
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch("requests.get",
mocked_response_edited), self.assertLogs(
level="INFO") as info_logs:
# Now proceed with the original queue_json_publish and call the
# up-to-date event for edited_url.
queue_json_publish(*args, **kwargs)
msg = Message.objects.select_related("sender").get(
id=msg_id)
self.assertIn(
f'<a href="{edited_url}" title="The Rock">The Rock</a>',
msg.rendered_content,
)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://edited.org/: "
in info_logs.output[0])
with mock_queue_publish("zerver.lib.actions.queue_json_publish",
wraps=wrapped_queue_json_publish):
result = self.client_patch(
"/json/messages/" + str(msg_id),
{
"message_id": msg_id,
"content": edited_url,
},
)
self.assert_json_success(result)
def _get_queued_data_for_message_update(
self, message_id: int, content: str, expect_short_circuit: bool = False
) -> Dict[str, Any]:
"""
This function updates a message with a post to
/json/messages/(message_id).
By using mocks, we are able to capture two pieces of data:
enqueue_kwargs: These are the arguments passed in to
maybe_enqueue_notifications.
queue_messages: These are the messages that
maybe_enqueue_notifications actually
puts on the queue.
Using this helper allows you to construct a test that goes
pretty deep into the missed-messages codepath, without actually
queuing the final messages.
"""
url = "/json/messages/" + str(message_id)
request = dict(
message_id=message_id,
content=content,
)
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as m:
result = self.client_patch(url, request)
cordelia = self.example_user("cordelia")
cordelia_calls = [
call_args
for call_args in m.call_args_list
if call_args[1]["user_notifications_data"].user_id == cordelia.id
]
if expect_short_circuit:
self.assert_length(cordelia_calls, 0)
return {}
# Normally we expect maybe_enqueue_notifications to be
# called for Cordelia, so continue on.
self.assert_length(cordelia_calls, 1)
enqueue_kwargs = cordelia_calls[0][1]
queue_messages = []
def fake_publish(queue_name: str, event: Union[Mapping[str, Any], str], *args: Any) -> None:
queue_messages.append(
dict(
queue_name=queue_name,
event=event,
)
)
with mock_queue_publish(
"zerver.tornado.event_queue.queue_json_publish", side_effect=fake_publish
) as m:
maybe_enqueue_notifications(**enqueue_kwargs)
self.assert_json_success(result)
return dict(
enqueue_kwargs=enqueue_kwargs,
queue_messages=queue_messages,
)
def test_report_error(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
self.make_stream("errors", user.realm)
params = fix_params(
dict(
message="hello",
stacktrace="trace",
ui_message=True,
user_agent="agent",
href="href",
log="log",
more_info=dict(foo="bar", draft_content="**draft**"),
))
subprocess_mock = mock.patch(
"zerver.views.report.subprocess.check_output",
side_effect=subprocess.CalledProcessError(1, []),
)
with mock_queue_publish("zerver.views.report.queue_json_publish"
) as m, subprocess_mock:
result = self.client_post("/json/report/error", params)
self.assert_json_success(result)
report = m.call_args[0][1]["report"]
for k in set(params) - {"ui_message", "more_info"}:
self.assertEqual(report[k], params[k])
self.assertEqual(report["more_info"],
dict(foo="bar", draft_content="'**xxxxx**'"))
self.assertEqual(report["user_email"], user.delivery_email)
# Teset with no more_info
del params["more_info"]
with mock_queue_publish("zerver.views.report.queue_json_publish"
) as m, subprocess_mock:
result = self.client_post("/json/report/error", params)
self.assert_json_success(result)
with self.settings(BROWSER_ERROR_REPORTING=False):
result = self.client_post("/json/report/error", params)
self.assert_json_success(result)
# If js_source_map is present, then the stack trace should be annotated.
# DEVELOPMENT=False and TEST_SUITE=False are necessary to ensure that
# js_source_map actually gets instantiated.
with self.settings(DEVELOPMENT=False, TEST_SUITE=False), mock.patch(
"zerver.lib.unminify.SourceMap.annotate_stacktrace"
) as annotate, self.assertLogs(level="INFO") as info_logs:
result = self.client_post("/json/report/error", params)
self.assert_json_success(result)
# fix_params (see above) adds quotes when JSON encoding.
annotate.assert_called_once_with('"trace"')
self.assertEqual(
info_logs.output,
["INFO:root:Processing traceback with type browser for None"])
# Now test without authentication.
self.logout()
with self.settings(DEVELOPMENT=False, TEST_SUITE=False), mock.patch(
"zerver.lib.unminify.SourceMap.annotate_stacktrace"
) as annotate, self.assertLogs(level="INFO") as info_logs:
result = self.client_post("/json/report/error", params)
self.assert_json_success(result)
self.assertEqual(
info_logs.output,
["INFO:root:Processing traceback with type browser for None"])
def test_request(self, mock_function: MagicMock) -> None:
mock_function.return_value = None
"""A normal request is handled properly"""
record = self.simulate_error()
assert isinstance(record, HasRequest)
report = self.run_handler(record)
self.assertIn("user", report)
self.assertIn("user_email", report["user"])
self.assertIn("user_role", report["user"])
self.assertIn("message", report)
self.assertIn("stack_trace", report)
# Test that `add_request_metadata` throwing an exception is fine
with patch("zerver.logging_handlers.traceback.print_exc"):
with patch(
"zerver.logging_handlers.add_request_metadata",
side_effect=Exception("Unexpected exception!"),
):
report = self.run_handler(record)
self.assertNotIn("user", report)
self.assertIn("message", report)
self.assertEqual(report["stack_trace"], "See /var/log/zulip/errors.log")
# Check anonymous user is handled correctly
record.request.user = AnonymousUser()
report = self.run_handler(record)
self.assertIn("host", report)
self.assertIn("user", report)
self.assertIn("user_email", report["user"])
self.assertIn("user_role", report["user"])
self.assertIn("message", report)
self.assertIn("stack_trace", report)
# Put it back so we continue to test the non-anonymous case
record.request.user = self.example_user("hamlet")
# Now simulate a DisallowedHost exception
def get_host_error() -> None:
raise Exception("Get host failure!")
orig_get_host = record.request.get_host
record.request.get_host = get_host_error
report = self.run_handler(record)
record.request.get_host = orig_get_host
self.assertIn("host", report)
self.assertIn("user", report)
self.assertIn("user_email", report["user"])
self.assertIn("user_role", report["user"])
self.assertIn("message", report)
self.assertIn("stack_trace", report)
# Test an exception_filter exception
with patch("zerver.logging_handlers.get_exception_reporter_filter", return_value=15):
record.request.method = "POST"
report = self.run_handler(record)
record.request.method = "GET"
self.assertIn("host", report)
self.assertIn("user", report)
self.assertIn("user_email", report["user"])
self.assertIn("user_role", report["user"])
self.assertIn("message", report)
self.assertIn("stack_trace", report)
# Test the catch-all exception handler doesn't throw
with patch(
"zerver.lib.error_notify.notify_server_error", side_effect=Exception("queue error")
):
self.handler.emit(record)
with self.settings(STAGING_ERROR_NOTIFICATIONS=False):
with mock_queue_publish(
"zerver.logging_handlers.queue_json_publish", side_effect=Exception("queue error")
) as m:
with patch("logging.warning") as log_mock:
self.handler.emit(record)
m.assert_called_once()
log_mock.assert_called_once_with(
"Reporting an exception triggered an exception!", exc_info=True
)
with mock_queue_publish("zerver.logging_handlers.queue_json_publish") as m:
with patch("logging.warning") as log_mock:
self.handler.emit(record)
m.assert_called_once()
log_mock.assert_not_called()
# Test no exc_info
record.exc_info = None
report = self.run_handler(record)
self.assertIn("host", report)
self.assertIn("user", report)
self.assertIn("user_email", report["user"])
self.assertIn("user_role", report["user"])
self.assertIn("message", report)
self.assertEqual(report["stack_trace"], "No stack trace available")
# Test arbitrary exceptions from request.user
record.request.user = None
with patch("zerver.logging_handlers.traceback.print_exc"):
report = self.run_handler(record)
self.assertIn("host", report)
self.assertIn("user", report)
self.assertIn("user_email", report["user"])
self.assertIn("user_role", report["user"])
self.assertIn("message", report)
self.assertIn("stack_trace", report)
def test_push_notifications_worker(self) -> None:
"""
The push notifications system has its own comprehensive test suite,
so we can limit ourselves to simple unit testing the queue processor,
without going deeper into the system - by mocking the handle_push_notification
functions to immediately produce the effect we want, to test its handling by the queue
processor.
"""
fake_client = FakeClient()
def fake_publish(
queue_name: str, event: Dict[str, Any], processor: Callable[[Any], None]
) -> None:
fake_client.enqueue(queue_name, event)
def generate_new_message_notification() -> Dict[str, Any]:
return build_offline_notification(1, 1)
def generate_remove_notification() -> Dict[str, Any]:
return {
"type": "remove",
"user_profile_id": 1,
"message_ids": [1],
}
with simulated_queue_client(fake_client):
worker = queue_processors.PushNotificationsWorker()
worker.setup()
with patch(
"zerver.worker.queue_processors.handle_push_notification"
) as mock_handle_new, patch(
"zerver.worker.queue_processors.handle_remove_push_notification"
) as mock_handle_remove, patch(
"zerver.worker.queue_processors.initialize_push_notifications"
):
event_new = generate_new_message_notification()
event_remove = generate_remove_notification()
fake_client.enqueue("missedmessage_mobile_notifications", event_new)
fake_client.enqueue("missedmessage_mobile_notifications", event_remove)
worker.start()
mock_handle_new.assert_called_once_with(event_new["user_profile_id"], event_new)
mock_handle_remove.assert_called_once_with(
event_remove["user_profile_id"], event_remove["message_ids"]
)
with patch(
"zerver.worker.queue_processors.handle_push_notification",
side_effect=PushNotificationBouncerRetryLaterError("test"),
) as mock_handle_new, patch(
"zerver.worker.queue_processors.handle_remove_push_notification",
side_effect=PushNotificationBouncerRetryLaterError("test"),
) as mock_handle_remove, patch(
"zerver.worker.queue_processors.initialize_push_notifications"
):
event_new = generate_new_message_notification()
event_remove = generate_remove_notification()
fake_client.enqueue("missedmessage_mobile_notifications", event_new)
fake_client.enqueue("missedmessage_mobile_notifications", event_remove)
with mock_queue_publish(
"zerver.lib.queue.queue_json_publish", side_effect=fake_publish
), self.assertLogs("zerver.worker.queue_processors", "WARNING") as warn_logs:
worker.start()
self.assertEqual(mock_handle_new.call_count, 1 + MAX_REQUEST_RETRIES)
self.assertEqual(mock_handle_remove.call_count, 1 + MAX_REQUEST_RETRIES)
self.assertEqual(
warn_logs.output,
[
"WARNING:zerver.worker.queue_processors:Maximum retries exceeded for trigger:1 event:push_notification",
]
* 2,
)
# This verifies the compatibility code for the `message_id` -> `message_ids`
# conversion for "remove" events.
with patch(
"zerver.worker.queue_processors.handle_remove_push_notification"
) as mock_handle_remove, patch(
"zerver.worker.queue_processors.initialize_push_notifications"
):
event_new = dict(
user_profile_id=10,
message_id=33,
type="remove",
)
fake_client.enqueue("missedmessage_mobile_notifications", event_new)
worker.start()
# The `message_id` field should have been converted to a list with a single element.
mock_handle_remove.assert_called_once_with(10, [33])