def test_invalid_url(self) -> None:
url = 'http://test.org/'
error_url = 'http://test.org/x'
with mock.patch('zerver.lib.actions.queue_json_publish'):
msg_id = self.send_personal_message(
self.example_email('hamlet'),
self.example_email('cordelia'),
content=error_url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
'message_id': msg_id,
'urls': [error_url],
'message_realm_id': msg.sender.realm_id,
'message_content': error_url}
mocked_response = mock.Mock(side_effect=self.create_mock_response(url))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response):
FetchLinksEmbedData().consume(event)
cached_data = link_embed_data_from_cache(error_url)
# FIXME: Should we really cache this, especially without cache invalidation?
self.assertIsNone(cached_data)
msg.refresh_from_db()
self.assertEqual(
'<p><a href="http://test.org/x" target="_blank" title="http://test.org/x">http://test.org/x</a></p>',
msg.rendered_content)
def test_link_preview_non_html_data(self) -> None:
email = self.example_email('hamlet')
self.login(email)
url = 'http://test.org/audio.mp3'
with mock.patch('zerver.lib.actions.queue_json_publish') as patched:
msg_id = self.send_stream_message(email, "Scotland", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
headers = {'content-type': 'application/octet-stream'}
mocked_response = mock.Mock(side_effect=self.create_mock_response(url, headers=headers))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response):
FetchLinksEmbedData().consume(event)
cached_data = link_embed_data_from_cache(url)
self.assertIsNone(cached_data)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertEqual(
('<p><a href="http://test.org/audio.mp3" target="_blank" title="http://test.org/audio.mp3">'
'http://test.org/audio.mp3</a></p>'),
msg.rendered_content)
def test_valid_content_type_error_get_data(self) -> None:
url = 'http://test.org/'
with mock.patch('zerver.lib.actions.queue_json_publish'):
msg_id = self.send_personal_message(
self.example_email('hamlet'),
self.example_email('cordelia'),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
'message_id': msg_id,
'urls': [url],
'message_realm_id': msg.sender.realm_id,
'message_content': url}
with mock.patch('zerver.lib.url_preview.preview.get_oembed_data', side_effect=lambda *args, **kwargs: None):
with mock.patch('zerver.lib.url_preview.preview.valid_content_type', side_effect=lambda k: True):
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mock.Mock(side_effect=ConnectionError())):
FetchLinksEmbedData().consume(event)
with self.assertRaises(NotFoundInCache):
link_embed_data_from_cache(url)
msg.refresh_from_db()
self.assertEqual(
'<p><a href="http://test.org/" target="_blank" title="http://test.org/">http://test.org/</a></p>',
msg.rendered_content)
def _send_message_with_test_org_url(self, sender_email: str, queue_should_run: bool=True,
relative_url: bool=False) -> Message:
url = 'http://test.org/'
with mock.patch('zerver.lib.actions.queue_json_publish') as patched:
msg_id = self.send_personal_message(
sender_email,
self.example_email('cordelia'),
content=url,
)
if queue_should_run:
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
else:
patched.assert_not_called()
# If we nothing was put in the queue, we don't need to
# run the queue processor or any of the following code
return Message.objects.select_related("sender").get(id=msg_id)
# Verify the initial message doesn't have the embedded links rendered
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertNotIn(
'<a href="{0}" target="_blank" title="The Rock">The Rock</a>'.format(url),
msg.rendered_content)
# Mock the network request result so the test can be fast without Internet
mocked_response = mock.Mock(side_effect=self.create_mock_response(url, relative_url=relative_url))
# Run the queue processor to potentially rerender things
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response):
FetchLinksEmbedData().consume(event)
msg = Message.objects.select_related("sender").get(id=msg_id)
return msg
def wrapped_queue_json_publish(*args: Any, **kwargs: Any) -> None:
# Mock the network request result so the test can be fast without Internet
mocked_response_original = mock.Mock(side_effect=self.create_mock_response(original_url))
mocked_response_edited = mock.Mock(side_effect=self.create_mock_response(edited_url))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response_original):
# Run the queue processor. This will simulate the event for original_url being
# processed after the message has been edited.
FetchLinksEmbedData().consume(event)
msg = Message.objects.select_related("sender").get(id=msg_id)
# The content of the message has changed since the event for original_url has been created,
# it should not be rendered. Another, up-to-date event will have been sent (edited_url).
self.assertNotIn('<a href="{0}" target="_blank" title="The Rock">The Rock</a>'.format(original_url),
msg.rendered_content)
mocked_response_edited.assert_not_called()
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response_edited):
# Now proceed with the original queue_json_publish and call the
# up-to-date event for edited_url.
queue_json_publish(*args, **kwargs)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertIn('<a href="{0}" target="_blank" title="The Rock">The Rock</a>'.format(edited_url),
msg.rendered_content)
def test_invalid_url(self) -> None:
url = "http://test.org/"
error_url = "http://test.org/x"
with mock_queue_publish("zerver.lib.actions.queue_json_publish"):
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=error_url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [error_url],
"message_realm_id": msg.sender.realm_id,
"message_content": error_url,
}
self.create_mock_response(error_url, status=404)
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/x: "
in info_logs.output[0])
cached_data = link_embed_data_from_cache(error_url)
# FIXME: Should we really cache this, especially without cache invalidation?
self.assertIsNone(cached_data)
msg.refresh_from_db()
self.assertEqual(
'<p><a href="http://test.org/x">http://test.org/x</a></p>',
msg.rendered_content)
self.assertTrue(responses.assert_call_count(url, 0))
def test_custom_title_replaces_youtube_url_title(self) -> None:
url = "[YouTube link](https://www.youtube.com/watch?v=eSJTXC7Ixgg)"
with mock_queue_publish("zerver.lib.actions.queue_json_publish"):
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [url],
"message_realm_id": msg.sender.realm_id,
"message_content": url,
}
mocked_data = {
"title":
"Clearer Code at Scale - Static Types at Zulip and Dropbox"
}
self.create_mock_response(url)
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
with mock.patch(
"zerver.lib.markdown.link_preview.link_embed_data_from_cache",
lambda *args, **kwargs: mocked_data,
):
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for [YouTube link](https://www.youtube.com/watch?v=eSJTXC7Ixgg):"
in info_logs.output[0])
msg.refresh_from_db()
expected_content = f"""<p><a href="https://www.youtube.com/watch?v=eSJTXC7Ixgg">YouTube link</a></p>\n<div class="youtube-video message_inline_image"><a data-id="eSJTXC7Ixgg" href="https://www.youtube.com/watch?v=eSJTXC7Ixgg"><img src="{get_camo_url("https://i.ytimg.com/vi/eSJTXC7Ixgg/default.jpg")}"></a></div>"""
self.assertEqual(expected_content, msg.rendered_content)
def test_link_preview_no_open_graph_image(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/foo.html"
with mock_queue_publish(
"zerver.lib.actions.queue_json_publish") as patched:
msg_id = self.send_stream_message(user,
"Scotland",
topic_name="foo",
content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
# HTML without the og:image metadata
html = "\n".join(line for line in self.open_graph_html.splitlines()
if "og:image" not in line)
self.create_mock_response(url, body=html)
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
cached_data = link_embed_data_from_cache(url)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/foo.html: "
in info_logs.output[0])
self.assertIn("title", cached_data)
self.assertNotIn("image", cached_data)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertEqual(
('<p><a href="http://test.org/foo.html">'
"http://test.org/foo.html</a></p>"),
msg.rendered_content,
)
def test_link_preview_no_content_type_header(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/"
with mock_queue_publish(
"zerver.lib.actions.queue_json_publish") as patched:
msg_id = self.send_stream_message(user,
"Scotland",
topic_name="foo",
content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
self.create_mock_response(url)
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
data = link_embed_data_from_cache(url)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0])
self.assertIn("title", data)
self.assertIn("image", data)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertIn(data["title"], msg.rendered_content)
self.assertIn(data["image"], msg.rendered_content)
def wrapped_queue_json_publish(*args: Any, **kwargs: Any) -> None:
self.create_mock_response(original_url)
self.create_mock_response(edited_url)
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
# Run the queue processor. This will simulate the event for original_url being
# processed after the message has been edited.
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0])
msg = Message.objects.select_related("sender").get(id=msg_id)
# The content of the message has changed since the event for original_url has been created,
# it should not be rendered. Another, up-to-date event will have been sent (edited_url).
self.assertNotIn(
f'<a href="{original_url}" title="The Rock">The Rock</a>',
msg.rendered_content)
self.assertTrue(responses.assert_call_count(edited_url, 0))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with self.assertLogs(level="INFO") as info_logs:
# Now proceed with the original queue_json_publish and call the
# up-to-date event for edited_url.
queue_json_publish(*args, **kwargs)
msg = Message.objects.select_related("sender").get(
id=msg_id)
self.assertIn(
f'<a href="{edited_url}" title="The Rock">The Rock</a>',
msg.rendered_content,
)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://edited.org/: "
in info_logs.output[0])
def test_youtube_url_title_replaces_url(self) -> None:
url = 'https://www.youtube.com/watch?v=eSJTXC7Ixgg'
with mock.patch('zerver.lib.actions.queue_json_publish'):
msg_id = self.send_personal_message(
self.example_email('hamlet'),
self.example_email('cordelia'),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
'message_id': msg_id,
'urls': [url],
'message_realm_id': msg.sender.realm_id,
'message_content': url}
mocked_data = {'title': 'Clearer Code at Scale - Static Types at Zulip and Dropbox'}
mocked_response = mock.Mock(side_effect=self.create_mock_response(url))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response):
with mock.patch('zerver.lib.bugdown.link_preview.link_embed_data_from_cache',
lambda *args, **kwargs: mocked_data):
FetchLinksEmbedData().consume(event)
msg.refresh_from_db()
expected_content = '<p><a href="https://www.youtube.com/watch?v=eSJTXC7Ixgg" target="_blank" title="https://www.youtube.com/watch?v=eSJTXC7Ixgg">YouTube - Clearer Code at Scale - Static Types at Zulip and Dropbox</a></p>\n<div class="youtube-video message_inline_image"><a data-id="eSJTXC7Ixgg" href="https://www.youtube.com/watch?v=eSJTXC7Ixgg" target="_blank" title="https://www.youtube.com/watch?v=eSJTXC7Ixgg"><img src="https://i.ytimg.com/vi/eSJTXC7Ixgg/default.jpg"></a></div>'
self.assertEqual(expected_content, msg.rendered_content)
def test_link_preview_no_open_graph_image(self) -> None:
email = self.example_email('hamlet')
self.login(email)
url = 'http://test.org/foo.html'
with mock.patch('zerver.lib.actions.queue_json_publish') as patched:
msg_id = self.send_stream_message(email, "Scotland", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
# HTML without the og:image metadata
html = '\n'.join(line for line in self.open_graph_html.splitlines() if 'og:image' not in line)
mocked_response = mock.Mock(side_effect=self.create_mock_response(url, html=html))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response):
FetchLinksEmbedData().consume(event)
cached_data = link_embed_data_from_cache(url)
self.assertIn('title', cached_data)
self.assertNotIn('image', cached_data)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertEqual(
('<p><a href="http://test.org/foo.html" target="_blank" title="http://test.org/foo.html">'
'http://test.org/foo.html</a></p>'),
msg.rendered_content)
def test_safe_oembed_html_url(self) -> None:
url = 'http://test.org/'
with mock.patch('zerver.lib.actions.queue_json_publish'):
msg_id = self.send_personal_message(
self.example_email('hamlet'),
self.example_email('cordelia'),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
'message_id': msg_id,
'urls': [url],
'message_realm_id': msg.sender.realm_id,
'message_content': url}
mocked_data = {'html': '<iframe src="{0}"></iframe>'.format(url),
'oembed': True, 'type': 'video', 'image': '{0}/image.png'.format(url)}
mocked_response = mock.Mock(side_effect=self.create_mock_response(url))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response):
with mock.patch('zerver.lib.url_preview.preview.get_oembed_data',
lambda *args, **kwargs: mocked_data):
FetchLinksEmbedData().consume(event)
data = link_embed_data_from_cache(url)
self.assertEqual(data, mocked_data)
msg.refresh_from_db()
self.assertIn('a data-id="{}"'.format(escape(mocked_data['html'])), msg.rendered_content)
def test_edit_message_history(self):
# type: () -> None
email = self.example_email('hamlet')
self.login(email)
msg_id = self.send_message(email,
"Scotland",
Recipient.STREAM,
subject="editing",
content="original")
url = 'http://test.org/'
response = MockPythonResponse(self.open_graph_html, 200)
mocked_response = mock.Mock(side_effect=lambda k: {url: response}.get(
k, MockPythonResponse('', 404)))
with mock.patch('zerver.views.messages.queue_json_publish') as patched:
result = self.client_patch("/json/messages/" + str(msg_id), {
'message_id': msg_id,
'content': url,
})
self.assert_json_success(result)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response):
FetchLinksEmbedData().consume(event)
embedded_link = '<a href="{0}" target="_blank" title="The Rock">The Rock</a>'.format(
url)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertIn(embedded_link, msg.rendered_content)
def test_link_preview_non_html_data(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/audio.mp3"
with mock_queue_publish("zerver.lib.actions.queue_json_publish") as patched:
msg_id = self.send_stream_message(user, "Scotland", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
headers = {"content-type": "application/octet-stream"}
mocked_response = mock.Mock(side_effect=self.create_mock_response(url, headers=headers))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch("requests.get", mocked_response), self.assertLogs(
level="INFO"
) as info_logs:
FetchLinksEmbedData().consume(event)
cached_data = link_embed_data_from_cache(url)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/audio.mp3: "
in info_logs.output[0]
)
self.assertIsNone(cached_data)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertEqual(
('<p><a href="http://test.org/audio.mp3">' "http://test.org/audio.mp3</a></p>"),
msg.rendered_content,
)
def test_link_preview_no_content_type_header(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/"
with mock_queue_publish("zerver.actions.message_send.queue_json_publish") as patched:
msg_id = self.send_stream_message(user, "Denmark", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
self.create_mock_response(url)
with self.settings(TEST_SUITE=False):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
cached_data = cache_get(preview_url_cache_key(url))[0]
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0]
)
assert cached_data is not None
msg = Message.objects.select_related("sender").get(id=msg_id)
assert msg.rendered_content is not None
self.assertIn(cached_data.title, msg.rendered_content)
assert cached_data.image is not None
self.assertIn(re.sub(r"([^\w-])", r"\\\1", cached_data.image), msg.rendered_content)
def test_link_preview_non_html_data(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/audio.mp3"
with mock_queue_publish("zerver.actions.message_send.queue_json_publish") as patched:
msg_id = self.send_stream_message(user, "Denmark", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
content_type = "application/octet-stream"
self.create_mock_response(url, content_type=content_type)
with self.settings(TEST_SUITE=False):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
cached_data = cache_get(preview_url_cache_key(url))[0]
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/audio.mp3: "
in info_logs.output[0]
)
self.assertIsNone(cached_data)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertEqual(
('<p><a href="http://test.org/audio.mp3">' "http://test.org/audio.mp3</a></p>"),
msg.rendered_content,
)
def test_link_preview_open_graph_image_missing_content(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/foo.html"
with mock_queue_publish("zerver.actions.message_send.queue_json_publish") as patched:
msg_id = self.send_stream_message(user, "Denmark", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
# HTML without the og:image metadata
html = "\n".join(
line if "og:image" not in line else '<meta property="og:image"/>'
for line in self.open_graph_html.splitlines()
)
self.create_mock_response(url, body=html)
with self.settings(TEST_SUITE=False):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
cached_data = cache_get(preview_url_cache_key(url))[0]
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/foo.html: "
in info_logs.output[0]
)
assert cached_data is not None
self.assertIsNotNone(cached_data.title)
self.assertIsNone(cached_data.image)
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertEqual(
('<p><a href="http://test.org/foo.html">' "http://test.org/foo.html</a></p>"),
msg.rendered_content,
)
def test_link_preview_css_escaping_image(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/"
with mock_queue_publish("zerver.actions.message_send.queue_json_publish") as patched:
msg_id = self.send_stream_message(user, "Denmark", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
# Swap the URL out for one with characters that need CSS escaping
html = re.sub(r"rock\.jpg", "rock).jpg", self.open_graph_html)
self.create_mock_response(url, body=html)
with self.settings(TEST_SUITE=False):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0]
)
msg = Message.objects.select_related("sender").get(id=msg_id)
with_preview = (
'<p><a href="http://test.org/">http://test.org/</a></p>\n<div class="message_embed"><a class="message_embed_image" href="http://test.org/" style="background-image: url('
+ "http\\:\\/\\/ia\\.media-imdb\\.com\\/images\\/rock\\)\\.jpg"
+ ')"></a><div class="data-container"><div class="message_embed_title"><a href="http://test.org/" title="The Rock">The Rock</a></div><div class="message_embed_description">Description text</div></div></div>'
)
self.assertEqual(
with_preview,
msg.rendered_content,
)
def test_http_error_get_data(self) -> None:
url = "http://test.org/"
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [url],
"message_realm_id": msg.sender.realm_id,
"message_content": url,
}
self.create_mock_response(url, body=ConnectionError())
with self.settings(INLINE_URL_EMBED_PREVIEW=True, TEST_SUITE=False):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0]
)
msg = Message.objects.get(id=msg_id)
self.assertEqual(
'<p><a href="http://test.org/">http://test.org/</a></p>', msg.rendered_content
)
def test_message_deleted(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
url = "http://test.org/"
with mock_queue_publish("zerver.actions.message_send.queue_json_publish") as patched:
msg_id = self.send_stream_message(user, "Denmark", topic_name="foo", content=url)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
msg = Message.objects.select_related("sender").get(id=msg_id)
do_delete_messages(msg.sender.realm, [msg])
# We do still fetch the URL, as we don't want to incur the
# cost of locking the row while we do the HTTP fetches.
self.create_mock_response(url)
with self.settings(TEST_SUITE=False):
with self.assertLogs(level="INFO") as info_logs:
# Run the queue processor. This will simulate the event for original_url being
# processed after the message has been deleted.
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0]
)
def test_edit_message_history(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
msg_id = self.send_stream_message(user, "Denmark", topic_name="editing", content="original")
url = "http://test.org/"
self.create_mock_response(url)
with mock_queue_publish("zerver.actions.message_edit.queue_json_publish") as patched:
result = self.client_patch(
"/json/messages/" + str(msg_id),
{
"content": url,
},
)
self.assert_json_success(result)
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
with self.settings(TEST_SUITE=False):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0]
)
embedded_link = f'<a href="{url}" title="The Rock">The Rock</a>'
msg = Message.objects.select_related("sender").get(id=msg_id)
assert msg.rendered_content is not None
self.assertIn(embedded_link, msg.rendered_content)
def test_http_error_get_data(self):
# type: () -> None
url = 'http://test.org/'
msg_id = self.send_message(self.example_email('hamlet'),
self.example_email('cordelia'),
Recipient.PERSONAL,
subject="url",
content=url)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
'message_id': msg_id,
'urls': [url],
'message_realm_id': msg.sender.realm_id,
'message_content': url
}
with self.settings(INLINE_URL_EMBED_PREVIEW=True,
TEST_SUITE=False,
CACHES=TEST_CACHES):
with mock.patch('requests.get',
mock.Mock(side_effect=ConnectionError())):
with mock.patch('logging.error') as error_mock:
FetchLinksEmbedData().consume(event)
self.assertEqual(error_mock.call_count, 1)
msg = Message.objects.get(id=msg_id)
self.assertEqual(
'<p><a href="http://test.org/" target="_blank" title="http://test.org/">http://test.org/</a></p>',
msg.rendered_content)
def _send_message_with_test_org_url(self,
sender_email,
queue_should_run=True,
relative_url=False):
# type: (str, bool, bool) -> Message
url = 'http://test.org/'
with mock.patch('zerver.lib.actions.queue_json_publish') as patched:
msg_id = self.send_message(sender_email,
self.example_email('cordelia'),
Recipient.PERSONAL,
subject="url",
content=url)
if queue_should_run:
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
else:
patched.assert_not_called()
# If we nothing was put in the queue, we don't need to
# run the queue processor or any of the following code
return Message.objects.select_related("sender").get(id=msg_id)
# Verify the initial message doesn't have the embedded links rendered
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertNotIn(
'<a href="{0}" target="_blank" title="The Rock">The Rock</a>'.
format(url), msg.rendered_content)
# Mock the network request result so the test can be fast without Internet
response = MockPythonResponse(self.open_graph_html, 200)
if relative_url is True:
response = MockPythonResponse(
self.open_graph_html.replace('http://ia.media-imdb.com', ''),
200)
mocked_response = mock.Mock(side_effect=lambda k: {url: response}.get(
k, MockPythonResponse('', 404)))
# Run the queue processor to potentially rerender things
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mocked_response):
FetchLinksEmbedData().consume(event)
msg = Message.objects.select_related("sender").get(id=msg_id)
return msg
def _send_message_with_test_org_url(self,
sender: UserProfile,
queue_should_run: bool = True,
relative_url: bool = False) -> Message:
url = "http://test.org/"
with mock_queue_publish(
"zerver.lib.actions.queue_json_publish") as patched:
msg_id = self.send_personal_message(
sender,
self.example_user("cordelia"),
content=url,
)
if queue_should_run:
patched.assert_called_once()
queue = patched.call_args[0][0]
self.assertEqual(queue, "embed_links")
event = patched.call_args[0][1]
else:
patched.assert_not_called()
# If we nothing was put in the queue, we don't need to
# run the queue processor or any of the following code
return Message.objects.select_related("sender").get(id=msg_id)
# Verify the initial message doesn't have the embedded links rendered
msg = Message.objects.select_related("sender").get(id=msg_id)
self.assertNotIn(f'<a href="{url}" title="The Rock">The Rock</a>',
msg.rendered_content)
# Mock the network request result so the test can be fast without Internet
mocked_response = mock.Mock(side_effect=self.create_mock_response(
url, relative_url=relative_url))
# Run the queue processor to potentially rerender things
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch("requests.get", mocked_response), self.assertLogs(
level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0])
msg = Message.objects.select_related("sender").get(id=msg_id)
return msg
def test_http_error_get_data(self) -> None:
url = 'http://test.org/'
msg_id = self.send_personal_message(
self.example_user('hamlet'),
self.example_user('cordelia'),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
'message_id': msg_id,
'urls': [url],
'message_realm_id': msg.sender.realm_id,
'message_content': url}
with self.settings(INLINE_URL_EMBED_PREVIEW=True, TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch('requests.get', mock.Mock(side_effect=ConnectionError())):
FetchLinksEmbedData().consume(event)
msg = Message.objects.get(id=msg_id)
self.assertEqual(
'<p><a href="http://test.org/" title="http://test.org/">http://test.org/</a></p>',
msg.rendered_content)
def test_valid_content_type_error_get_data(self) -> None:
url = "http://test.org/"
with mock_queue_publish(
"zerver.actions.message_send.queue_json_publish"):
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [url],
"message_realm_id": msg.sender.realm_id,
"message_content": url,
}
self.create_mock_response(url, body=ConnectionError())
with mock.patch(
"zerver.lib.url_preview.preview.get_oembed_data",
side_effect=lambda *args, **kwargs: None,
):
with mock.patch(
"zerver.lib.url_preview.preview.valid_content_type",
side_effect=lambda k: True):
with self.settings(TEST_SUITE=False):
with self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0])
# This did not get cached -- hence the lack of [0] on the cache_get
cached_data = cache_get(preview_url_cache_key(url))
self.assertIsNone(cached_data)
msg.refresh_from_db()
self.assertEqual(
'<p><a href="http://test.org/">http://test.org/</a></p>',
msg.rendered_content)
def test_safe_oembed_html_url(self) -> None:
url = "http://test.org/"
with mock_queue_publish("zerver.lib.actions.queue_json_publish"):
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [url],
"message_realm_id": msg.sender.realm_id,
"message_content": url,
}
mocked_data = {
"html": f'<iframe src="{url}"></iframe>',
"oembed": True,
"type": "video",
"image": f"{url}/image.png",
}
mocked_response = mock.Mock(side_effect=self.create_mock_response(url))
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch("requests.get", mocked_response), self.assertLogs(
level="INFO"
) as info_logs:
with mock.patch(
"zerver.lib.url_preview.preview.get_oembed_data",
lambda *args, **kwargs: mocked_data,
):
FetchLinksEmbedData().consume(event)
data = link_embed_data_from_cache(url)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0]
)
self.assertEqual(data, mocked_data)
msg.refresh_from_db()
self.assertIn('a data-id="{}"'.format(escape(mocked_data["html"])), msg.rendered_content)
def test_valid_content_type_error_get_data(self) -> None:
url = "http://test.org/"
with mock_queue_publish("zerver.lib.actions.queue_json_publish"):
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [url],
"message_realm_id": msg.sender.realm_id,
"message_content": url,
}
with mock.patch(
"zerver.lib.url_preview.preview.get_oembed_data",
side_effect=lambda *args, **kwargs: None,
):
with mock.patch(
"zerver.lib.url_preview.preview.valid_content_type",
side_effect=lambda k: True):
with self.settings(TEST_SUITE=False, CACHES=TEST_CACHES):
with mock.patch(
"requests.get",
mock.Mock(side_effect=ConnectionError(
))), self.assertLogs(level="INFO") as info_logs:
FetchLinksEmbedData().consume(event)
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0])
with self.assertRaises(NotFoundInCache):
link_embed_data_from_cache(url)
msg.refresh_from_db()
self.assertEqual(
'<p><a href="http://test.org/">http://test.org/</a></p>',
msg.rendered_content)
def test_safe_oembed_html_url(self) -> None:
url = "http://test.org/"
with mock_queue_publish(
"zerver.actions.message_send.queue_json_publish"):
msg_id = self.send_personal_message(
self.example_user("hamlet"),
self.example_user("cordelia"),
content=url,
)
msg = Message.objects.select_related("sender").get(id=msg_id)
event = {
"message_id": msg_id,
"urls": [url],
"message_realm_id": msg.sender.realm_id,
"message_content": url,
}
mocked_data = UrlOEmbedData(
html=f'<iframe src="{url}"></iframe>',
type="video",
image=f"{url}/image.png",
)
self.create_mock_response(url)
with self.settings(TEST_SUITE=False):
with self.assertLogs(level="INFO") as info_logs:
with mock.patch(
"zerver.lib.url_preview.preview.get_oembed_data",
lambda *args, **kwargs: mocked_data,
):
FetchLinksEmbedData().consume(event)
cached_data = cache_get(preview_url_cache_key(url))[0]
self.assertTrue(
"INFO:root:Time spent on get_link_embed_data for http://test.org/: "
in info_logs.output[0])
self.assertEqual(cached_data, mocked_data)
msg.refresh_from_db()
self.assertIn(f'a data-id="{escape(mocked_data.html)}"',
msg.rendered_content)