def test_display_recipient_various_types(self) -> None:
hamlet = self.example_user('hamlet')
cordelia = self.example_user('cordelia')
othello = self.example_user('othello')
iago = self.example_user('iago')
message_ids = [
self.send_huddle_message(hamlet, [cordelia, othello], 'test'),
self.send_stream_message(cordelia, "Verona", content='test'),
self.send_personal_message(hamlet, cordelia, 'test'),
self.send_stream_message(cordelia, "Denmark", content='test'),
self.send_huddle_message(cordelia, [hamlet, othello, iago], 'test'),
self.send_personal_message(cordelia, othello, 'test'),
]
messages = messages_for_ids(
message_ids=message_ids,
user_message_flags={message_id: ['read'] for message_id in message_ids},
search_fields={},
apply_markdown=True,
client_gravatar=True,
allow_edit_history=False,
)
self._verify_display_recipient(messages[0]['display_recipient'], [hamlet, cordelia, othello])
self._verify_display_recipient(messages[1]['display_recipient'], get_stream("Verona", hamlet.realm))
self._verify_display_recipient(messages[2]['display_recipient'], [hamlet, cordelia])
self._verify_display_recipient(messages[3]['display_recipient'], get_stream("Denmark", hamlet.realm))
self._verify_display_recipient(messages[4]['display_recipient'], [hamlet, cordelia, othello, iago])
self._verify_display_recipient(messages[5]['display_recipient'], [cordelia, othello])
def test_display_recipient_huddle(self) -> None:
hamlet = self.example_user('hamlet')
cordelia = self.example_user('cordelia')
othello = self.example_user('othello')
iago = self.example_user('iago')
message_ids = [
self.send_huddle_message(hamlet, [cordelia, othello], 'test'),
self.send_huddle_message(cordelia, [hamlet, othello, iago],
'test'),
]
messages = messages_for_ids(
message_ids=message_ids,
user_message_flags={
message_id: ['read']
for message_id in message_ids
},
search_fields={},
apply_markdown=True,
client_gravatar=True,
allow_edit_history=False,
)
self._verify_display_recipient(messages[0]['display_recipient'],
[hamlet, cordelia, othello])
self._verify_display_recipient(messages[1]['display_recipient'],
[hamlet, cordelia, othello, iago])
def test_display_recipient_huddle(self) -> None:
hamlet = self.example_user("hamlet")
cordelia = self.example_user("cordelia")
othello = self.example_user("othello")
iago = self.example_user("iago")
message_ids = [
self.send_huddle_message(hamlet, [cordelia, othello], "test"),
self.send_huddle_message(cordelia, [hamlet, othello, iago],
"test"),
]
messages = messages_for_ids(
message_ids=message_ids,
user_message_flags={
message_id: ["read"]
for message_id in message_ids
},
search_fields={},
apply_markdown=True,
client_gravatar=True,
allow_edit_history=False,
)
self._verify_display_recipient(messages[0]["display_recipient"],
[hamlet, cordelia, othello])
self._verify_display_recipient(messages[1]["display_recipient"],
[hamlet, cordelia, othello, iago])
def test_display_recipient_stream(self) -> None:
cordelia = self.example_user("cordelia")
self.subscribe(cordelia, "Denmark")
message_ids = [
self.send_stream_message(cordelia, "Verona", content="test"),
self.send_stream_message(cordelia, "Denmark", content="test"),
]
messages = messages_for_ids(
message_ids=message_ids,
user_message_flags={
message_id: ["read"]
for message_id in message_ids
},
search_fields={},
apply_markdown=True,
client_gravatar=True,
allow_edit_history=False,
)
self._verify_display_recipient(messages[0]["display_recipient"],
get_stream("Verona", cordelia.realm))
self._verify_display_recipient(messages[1]["display_recipient"],
get_stream("Denmark", cordelia.realm))
def test_display_recipient_up_to_date(self) -> None:
"""
This is a test for a bug where due to caching of message_dicts,
after updating a user's information, fetching those cached messages
via messages_for_ids would return message_dicts with display_recipient
still having the old information. The returned message_dicts should have
up-to-date display_recipients and we check for that here.
"""
hamlet = self.example_user("hamlet")
cordelia = self.example_user("cordelia")
message_id = self.send_personal_message(hamlet, cordelia, "test")
cordelia_recipient = cordelia.recipient
# Cause the display_recipient to get cached:
assert cordelia_recipient is not None
get_display_recipient(cordelia_recipient)
# Change cordelia's email:
cordelia_new_email = "*****@*****.**"
cordelia.email = cordelia_new_email
cordelia.save()
# Local display_recipient cache needs to be flushed.
# flush_per_request_caches() is called after every request,
# so it makes sense to run it here.
flush_per_request_caches()
messages = messages_for_ids(
message_ids=[message_id],
user_message_flags={message_id: ["read"]},
search_fields={},
apply_markdown=True,
client_gravatar=True,
allow_edit_history=False,
)
message = messages[0]
# Find which display_recipient in the list is cordelia:
for display_recipient in message["display_recipient"]:
if display_recipient["id"] == cordelia.id:
cordelia_display_recipient = display_recipient
# Make sure the email is up-to-date.
self.assertEqual(cordelia_display_recipient["email"],
cordelia_new_email)
def test_messages_for_ids(self) -> None:
hamlet = self.example_user('hamlet')
cordelia = self.example_user('cordelia')
stream_name = 'test stream'
self.subscribe(cordelia, stream_name)
old_message_id = self.send_stream_message(cordelia,
stream_name,
content='foo')
self.subscribe(hamlet, stream_name)
content = 'hello @**King Hamlet**'
new_message_id = self.send_stream_message(cordelia,
stream_name,
content=content)
user_message_flags = {
old_message_id: ['read', 'historical'],
new_message_id: ['mentioned'],
}
messages = messages_for_ids(
message_ids=[old_message_id, new_message_id],
user_message_flags=user_message_flags,
search_fields={},
apply_markdown=True,
client_gravatar=True,
allow_edit_history=False,
)
self.assertEqual(len(messages), 2)
for message in messages:
if message['id'] == old_message_id:
old_message = message
elif message['id'] == new_message_id:
new_message = message
self.assertEqual(old_message['content'], '<p>foo</p>')
self.assertEqual(old_message['flags'], ['read', 'historical'])
self.assertIn('class="user-mention"', new_message['content'])
self.assertEqual(new_message['flags'], ['mentioned'])
def test_messages_for_ids(self) -> None:
hamlet = self.example_user("hamlet")
cordelia = self.example_user("cordelia")
stream_name = "test stream"
self.subscribe(cordelia, stream_name)
old_message_id = self.send_stream_message(cordelia,
stream_name,
content="foo")
self.subscribe(hamlet, stream_name)
content = "hello @**King Hamlet**"
new_message_id = self.send_stream_message(cordelia,
stream_name,
content=content)
user_message_flags = {
old_message_id: ["read", "historical"],
new_message_id: ["mentioned"],
}
messages = messages_for_ids(
message_ids=[old_message_id, new_message_id],
user_message_flags=user_message_flags,
search_fields={},
apply_markdown=True,
client_gravatar=True,
allow_edit_history=False,
)
self.assert_length(messages, 2)
for message in messages:
if message["id"] == old_message_id:
old_message = message
elif message["id"] == new_message_id:
new_message = message
self.assertEqual(old_message["content"], "<p>foo</p>")
self.assertEqual(old_message["flags"], ["read", "historical"])
self.assertIn('class="user-mention"', new_message["content"])
self.assertEqual(new_message["flags"], ["mentioned"])
def test_display_recipient_various_types(self) -> None:
hamlet = self.example_user("hamlet")
cordelia = self.example_user("cordelia")
othello = self.example_user("othello")
iago = self.example_user("iago")
self.subscribe(cordelia, "Denmark")
self.subscribe(hamlet, "Scotland")
message_ids = [
self.send_huddle_message(hamlet, [cordelia, othello], "test"),
self.send_stream_message(cordelia, "Verona", content="test"),
self.send_personal_message(hamlet, cordelia, "test"),
self.send_stream_message(cordelia, "Denmark", content="test"),
self.send_huddle_message(cordelia, [hamlet, othello, iago],
"test"),
self.send_personal_message(cordelia, othello, "test"),
]
messages = messages_for_ids(
message_ids=message_ids,
user_message_flags={
message_id: ["read"]
for message_id in message_ids
},
search_fields={},
apply_markdown=True,
client_gravatar=True,
allow_edit_history=False,
)
self._verify_display_recipient(messages[0]["display_recipient"],
[hamlet, cordelia, othello])
self._verify_display_recipient(messages[1]["display_recipient"],
get_stream("Verona", hamlet.realm))
self._verify_display_recipient(messages[2]["display_recipient"],
[hamlet, cordelia])
self._verify_display_recipient(messages[3]["display_recipient"],
get_stream("Denmark", hamlet.realm))
self._verify_display_recipient(messages[4]["display_recipient"],
[hamlet, cordelia, othello, iago])
self._verify_display_recipient(messages[5]["display_recipient"],
[cordelia, othello])
def json_fetch_raw_message(
request: HttpRequest,
maybe_user_profile: Union[UserProfile, AnonymousUser],
message_id: int = REQ(converter=to_non_negative_int, path_only=True),
apply_markdown: bool = REQ(json_validator=check_bool, default=True),
) -> HttpResponse:
if not maybe_user_profile.is_authenticated:
realm = get_valid_realm_from_request(request)
message = access_web_public_message(realm, message_id)
else:
(message, user_message) = access_message(maybe_user_profile,
message_id)
flags = ["read"]
if not maybe_user_profile.is_authenticated:
allow_edit_history = realm.allow_edit_history
else:
if user_message:
flags = user_message.flags_list()
else:
flags = ["read", "historical"]
allow_edit_history = maybe_user_profile.realm.allow_edit_history
# Security note: It's important that we call this only with a
# message already fetched via `access_message` type methods,
# as we do above.
message_dict_list = messages_for_ids(
message_ids=[message.id],
user_message_flags={message_id: flags},
search_fields={},
apply_markdown=apply_markdown,
client_gravatar=True,
allow_edit_history=allow_edit_history,
)
response = dict(
message=message_dict_list[0],
# raw_content is deprecated; we will need to wait until
# clients have been fully migrated to using the modern API
# before removing this, probably in 2023.
raw_content=message.content,
)
return json_success(request, response)
def get_messages_backend(request: HttpRequest, user_profile: UserProfile,
anchor_val: Optional[str]=REQ(
'anchor', str_validator=check_string, default=None),
num_before: int=REQ(converter=to_non_negative_int),
num_after: int=REQ(converter=to_non_negative_int),
narrow: OptionalNarrowListT=REQ('narrow', converter=narrow_parameter, default=None),
use_first_unread_anchor_val: bool=REQ('use_first_unread_anchor',
validator=check_bool, default=False),
client_gravatar: bool=REQ(validator=check_bool, default=False),
apply_markdown: bool=REQ(validator=check_bool, default=True)) -> HttpResponse:
anchor = parse_anchor_value(anchor_val, use_first_unread_anchor_val)
if num_before + num_after > MAX_MESSAGES_PER_FETCH:
return json_error(_("Too many messages requested (maximum {}).").format(
MAX_MESSAGES_PER_FETCH,
))
if user_profile.realm.email_address_visibility != Realm.EMAIL_ADDRESS_VISIBILITY_EVERYONE:
# If email addresses are only available to administrators,
# clients cannot compute gravatars, so we force-set it to false.
client_gravatar = False
include_history = ok_to_include_history(narrow, user_profile)
if include_history:
# The initial query in this case doesn't use `zerver_usermessage`,
# and isn't yet limited to messages the user is entitled to see!
#
# This is OK only because we've made sure this is a narrow that
# will cause us to limit the query appropriately later.
# See `ok_to_include_history` for details.
need_message = True
need_user_message = False
elif narrow is None:
# We need to limit to messages the user has received, but we don't actually
# need any fields from Message
need_message = False
need_user_message = True
else:
need_message = True
need_user_message = True
query, inner_msg_id_col = get_base_query_for_search(
user_profile=user_profile,
need_message=need_message,
need_user_message=need_user_message,
)
query, is_search = add_narrow_conditions(
user_profile=user_profile,
inner_msg_id_col=inner_msg_id_col,
query=query,
narrow=narrow,
)
if narrow is not None:
# Add some metadata to our logging data for narrows
verbose_operators = []
for term in narrow:
if term['operator'] == "is":
verbose_operators.append("is:" + term['operand'])
else:
verbose_operators.append(term['operator'])
request._log_data['extra'] = "[{}]".format(",".join(verbose_operators))
sa_conn = get_sqlalchemy_connection()
if anchor is None:
# The use_first_unread_anchor code path
anchor = find_first_unread_anchor(
sa_conn,
user_profile,
narrow,
)
anchored_to_left = (anchor == 0)
# Set value that will be used to short circuit the after_query
# altogether and avoid needless conditions in the before_query.
anchored_to_right = (anchor >= LARGER_THAN_MAX_MESSAGE_ID)
if anchored_to_right:
num_after = 0
first_visible_message_id = get_first_visible_message_id(user_profile.realm)
query = limit_query_to_range(
query=query,
num_before=num_before,
num_after=num_after,
anchor=anchor,
anchored_to_left=anchored_to_left,
anchored_to_right=anchored_to_right,
id_col=inner_msg_id_col,
first_visible_message_id=first_visible_message_id,
)
main_query = alias(query)
query = select(main_query.c, None, main_query).order_by(column("message_id").asc())
# This is a hack to tag the query we use for testing
query = query.prefix_with("/* get_messages */")
rows = list(sa_conn.execute(query).fetchall())
query_info = post_process_limited_query(
rows=rows,
num_before=num_before,
num_after=num_after,
anchor=anchor,
anchored_to_left=anchored_to_left,
anchored_to_right=anchored_to_right,
first_visible_message_id=first_visible_message_id,
)
rows = query_info['rows']
# The following is a little messy, but ensures that the code paths
# are similar regardless of the value of include_history. The
# 'user_messages' dictionary maps each message to the user's
# UserMessage object for that message, which we will attach to the
# rendered message dict before returning it. We attempt to
# bulk-fetch rendered message dicts from remote cache using the
# 'messages' list.
message_ids: List[int] = []
user_message_flags: Dict[int, List[str]] = {}
if include_history:
message_ids = [row[0] for row in rows]
# TODO: This could be done with an outer join instead of two queries
um_rows = UserMessage.objects.filter(user_profile=user_profile,
message__id__in=message_ids)
user_message_flags = {um.message_id: um.flags_list() for um in um_rows}
for message_id in message_ids:
if message_id not in user_message_flags:
user_message_flags[message_id] = ["read", "historical"]
else:
for row in rows:
message_id = row[0]
flags = row[1]
user_message_flags[message_id] = UserMessage.flags_list_for_flags(flags)
message_ids.append(message_id)
search_fields: Dict[int, Dict[str, str]] = dict()
if is_search:
for row in rows:
message_id = row[0]
(topic_name, rendered_content, content_matches, topic_matches) = row[-4:]
try:
search_fields[message_id] = get_search_fields(rendered_content, topic_name,
content_matches, topic_matches)
except UnicodeDecodeError as err: # nocoverage
# No coverage for this block since it should be
# impossible, and we plan to remove it once we've
# debugged the case that makes it happen.
raise Exception(str(err), message_id, narrow)
message_list = messages_for_ids(
message_ids=message_ids,
user_message_flags=user_message_flags,
search_fields=search_fields,
apply_markdown=apply_markdown,
client_gravatar=client_gravatar,
allow_edit_history=user_profile.realm.allow_edit_history,
)
statsd.incr('loaded_old_messages', len(message_list))
ret = dict(
messages=message_list,
result='success',
msg='',
found_anchor=query_info['found_anchor'],
found_oldest=query_info['found_oldest'],
found_newest=query_info['found_newest'],
history_limited=query_info['history_limited'],
anchor=anchor,
)
return json_success(ret)
