def render_markdown(message,
content,
realm=None,
realm_alert_words=None,
message_users=None):
# type: (Message, Text, Optional[Realm], Optional[RealmAlertWords], Set[UserProfile]) -> Text
"""Return HTML for given markdown. Bugdown may add properties to the
message object such as `mentions_user_ids` and `mentions_wildcard`.
These are only on this Django object and are not saved in the
database.
"""
if message_users is None:
message_user_ids = set() # type: Set[int]
else:
message_user_ids = {u.id for u in message_users}
if message is not None:
message.mentions_wildcard = False
message.is_me_message = False
message.mentions_user_ids = set()
message.alert_words = set()
message.links_for_preview = set()
if realm is None:
realm = message.get_realm()
possible_words = set() # type: Set[Text]
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
possible_words.update(set(words))
if message is None:
# If we don't have a message, then we are in the compose preview
# codepath, so we know we are dealing with a human.
sent_by_bot = False
else:
sent_by_bot = get_user_profile_by_id(message.sender_id).is_bot
# DO MAIN WORK HERE -- call bugdown to convert
rendered_content = bugdown.convert(content,
message=message,
message_realm=realm,
possible_words=possible_words,
sent_by_bot=sent_by_bot)
if message is not None:
message.user_ids_with_alert_words = set()
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
if set(words).intersection(message.alert_words):
message.user_ids_with_alert_words.add(user_id)
message.is_me_message = Message.is_status_message(
content, rendered_content)
return rendered_content
def archive(request: HttpRequest, stream_id: int, topic_name: str) -> HttpResponse:
def get_response(
rendered_message_list: List[str], is_web_public: bool, stream_name: str
) -> HttpResponse:
return render(
request,
"zerver/archive/index.html",
context={
"is_web_public": is_web_public,
"message_list": rendered_message_list,
"stream": stream_name,
"topic": topic_name,
},
)
try:
stream = access_web_public_stream(stream_id, request.realm)
except JsonableError:
return get_response([], False, "")
all_messages = list(
messages_for_topic(
stream_recipient_id=stream.recipient_id,
topic_name=topic_name,
)
.select_related("sender")
.order_by("date_sent"),
)
if not all_messages:
return get_response([], True, stream.name)
rendered_message_list = []
prev_sender: Optional[UserProfile] = None
for msg in all_messages:
include_sender = False
status_message = Message.is_status_message(msg.content, msg.rendered_content)
if not prev_sender or prev_sender != msg.sender or status_message:
if status_message:
prev_sender = None
else:
prev_sender = msg.sender
include_sender = True
if status_message:
status_message = msg.rendered_content[4 + 3 : -4]
context = {
"sender_full_name": msg.sender.full_name,
"timestampstr": datetime_to_timestamp(
msg.last_edit_time if msg.last_edit_time else msg.date_sent
),
"message_content": msg.rendered_content,
"avatar_url": get_gravatar_url(msg.sender.delivery_email, 1),
"include_sender": include_sender,
"status_message": status_message,
}
rendered_msg = loader.render_to_string("zerver/archive/single_message.html", context)
rendered_message_list.append(rendered_msg)
return get_response(rendered_message_list, True, stream.name)
def archive(request: HttpRequest,
stream_id: int,
topic_name: str) -> HttpResponse:
def get_response(rendered_message_list: List[str],
is_web_public: bool,
stream_name: str) -> HttpResponse:
return render(
request,
'zerver/archive/index.html',
context={
'is_web_public': is_web_public,
'message_list': rendered_message_list,
'stream': stream_name,
'topic': topic_name,
}
)
try:
stream = get_stream_by_id(stream_id)
except JsonableError:
return get_response([], False, '')
if not stream.is_web_public:
return get_response([], False, '')
all_messages = list(Message.objects.select_related(
'sender').filter(recipient__type_id=stream_id, subject=topic_name).order_by('pub_date'))
if not all_messages:
return get_response([], True, stream.name)
rendered_message_list = []
prev_sender = None
for msg in all_messages:
include_sender = False
status_message = Message.is_status_message(msg.content, msg.rendered_content)
if not prev_sender or prev_sender != msg.sender or status_message:
if status_message:
prev_sender = None
else:
prev_sender = msg.sender
include_sender = True
if status_message:
status_message = msg.rendered_content[4+3: -4]
context = {
'sender_full_name': msg.sender.full_name,
'timestampstr': datetime_to_timestamp(msg.last_edit_time
if msg.last_edit_time
else msg.pub_date),
'message_content': msg.rendered_content,
'avatar_url': get_gravatar_url(msg.sender.email, 1),
'include_sender': include_sender,
'status_message': status_message,
}
rendered_msg = loader.render_to_string('zerver/archive/single_message.html', context)
rendered_message_list.append(rendered_msg)
return get_response(rendered_message_list, True, stream.name)
def archive(request: HttpRequest,
stream_id: int,
topic_name: str) -> HttpResponse:
def get_response(rendered_message_list: List[str],
is_web_public: bool,
stream_name: str) -> HttpResponse:
return render(
request,
'zerver/archive/index.html',
context={
'is_web_public': is_web_public,
'message_list': rendered_message_list,
'stream': stream_name,
'topic': topic_name,
}
)
try:
stream = get_stream_by_id(stream_id)
except JsonableError:
return get_response([], False, '')
if not stream.is_web_public:
return get_response([], False, '')
all_messages = list(Message.objects.select_related(
'sender').filter(recipient__type_id=stream_id, subject=topic_name).order_by('pub_date'))
if not all_messages:
return get_response([], True, stream.name)
rendered_message_list = []
prev_sender = None
for msg in all_messages:
include_sender = False
status_message = Message.is_status_message(msg.content, msg.rendered_content)
if not prev_sender or prev_sender != msg.sender or status_message:
if status_message:
prev_sender = None
else:
prev_sender = msg.sender
include_sender = True
if status_message:
status_message = msg.rendered_content[4+3: -4]
context = {
'sender_full_name': msg.sender.full_name,
'timestampstr': datetime_to_timestamp(msg.last_edit_time
if msg.last_edit_time
else msg.pub_date),
'message_content': msg.rendered_content,
'avatar_url': get_gravatar_url(msg.sender.email, 1),
'include_sender': include_sender,
'status_message': status_message,
}
rendered_msg = loader.render_to_string('zerver/archive/single_message.html', context)
rendered_message_list.append(rendered_msg)
return get_response(rendered_message_list, True, stream.name)
def render_markdown(message,
content,
realm_id=None,
realm_alert_words=None,
message_users=None):
# type: (Message, Text, Optional[int], Optional[RealmAlertWords], Set[UserProfile]) -> Text
"""Return HTML for given markdown. Bugdown may add properties to the
message object such as `mentions_user_ids` and `mentions_wildcard`.
These are only on this Django object and are not saved in the
database.
"""
if message_users is None:
message_user_ids = set() # type: Set[int]
else:
message_user_ids = {u.id for u in message_users}
if message is not None:
message.mentions_wildcard = False
message.is_me_message = False
message.mentions_user_ids = set()
message.alert_words = set()
message.links_for_preview = set()
if realm_id is None:
realm_id = message.sender.realm_id
if message.sending_client.name == "zephyr_mirror" and message.sender.realm.is_zephyr_mirror_realm:
# Use slightly customized Markdown processor for content
# delivered via zephyr_mirror
realm_id = bugdown.ZEPHYR_MIRROR_BUGDOWN_KEY
possible_words = set() # type: Set[Text]
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
possible_words.update(set(words))
# DO MAIN WORK HERE -- call bugdown to convert
rendered_content = bugdown.convert(content,
realm_filters_key=realm_id,
message=message,
possible_words=possible_words)
if message is not None:
message.user_ids_with_alert_words = set()
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
if set(words).intersection(message.alert_words):
message.user_ids_with_alert_words.add(user_id)
message.is_me_message = Message.is_status_message(
content, rendered_content)
return rendered_content
def render_markdown(message, content, realm=None, realm_alert_words=None, message_users=None):
# type: (Message, Text, Optional[Realm], Optional[RealmAlertWords], Set[UserProfile]) -> Text
"""Return HTML for given markdown. Bugdown may add properties to the
message object such as `mentions_user_ids` and `mentions_wildcard`.
These are only on this Django object and are not saved in the
database.
"""
if message_users is None:
message_user_ids = set() # type: Set[int]
else:
message_user_ids = {u.id for u in message_users}
if message is not None:
message.mentions_wildcard = False
message.is_me_message = False
message.mentions_user_ids = set()
message.alert_words = set()
message.links_for_preview = set()
message.outgoing_webhook_bot_triggers = []
if realm is None:
realm = message.get_realm()
possible_words = set() # type: Set[Text]
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
possible_words.update(set(words))
if message is None:
# If we don't have a message, then we are in the compose preview
# codepath, so we know we are dealing with a human.
sent_by_bot = False
else:
sent_by_bot = get_user_profile_by_id(message.sender_id).is_bot
# DO MAIN WORK HERE -- call bugdown to convert
rendered_content = bugdown.convert(content, message=message, message_realm=realm,
possible_words=possible_words,
sent_by_bot=sent_by_bot)
if message is not None:
message.user_ids_with_alert_words = set()
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
if set(words).intersection(message.alert_words):
message.user_ids_with_alert_words.add(user_id)
message.is_me_message = Message.is_status_message(content, rendered_content)
return rendered_content
def render_markdown(message, content, realm_id=None, realm_alert_words=None, message_users=None):
# type: (Message, Text, Optional[int], Optional[RealmAlertWords], Set[UserProfile]) -> Text
"""Return HTML for given markdown. Bugdown may add properties to the
message object such as `mentions_user_ids` and `mentions_wildcard`.
These are only on this Django object and are not saved in the
database.
"""
if message_users is None:
message_user_ids = set() # type: Set[int]
else:
message_user_ids = {u.id for u in message_users}
if message is not None:
message.mentions_wildcard = False
message.is_me_message = False
message.mentions_user_ids = set()
message.alert_words = set()
message.links_for_preview = set()
if realm_id is None:
realm_id = message.sender.realm_id
if message.sending_client.name == "zephyr_mirror" and message.sender.realm.is_zephyr_mirror_realm:
# Use slightly customized Markdown processor for content
# delivered via zephyr_mirror
realm_id = bugdown.ZEPHYR_MIRROR_BUGDOWN_KEY
possible_words = set() # type: Set[Text]
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
possible_words.update(set(words))
# DO MAIN WORK HERE -- call bugdown to convert
rendered_content = bugdown.convert(content, realm_filters_key=realm_id, message=message,
possible_words=possible_words)
if message is not None:
message.user_ids_with_alert_words = set()
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
if set(words).intersection(message.alert_words):
message.user_ids_with_alert_words.add(user_id)
message.is_me_message = Message.is_status_message(content, rendered_content)
return rendered_content
def build_message_dict(
message: Optional[Message], message_id: int,
last_edit_time: Optional[datetime.datetime],
edit_history: Optional[str], content: str, topic_name: str,
date_sent: datetime.datetime, rendered_content: Optional[str],
rendered_content_version: Optional[int], sender_id: int,
sender_realm_id: int, sending_client_name: str, recipient_id: int,
recipient_type: int, recipient_type_id: int,
reactions: List[Dict[str, Any]],
submessages: List[Dict[str, Any]]) -> Dict[str, Any]:
obj = dict(id=message_id,
sender_id=sender_id,
content=content,
recipient_type_id=recipient_type_id,
recipient_type=recipient_type,
recipient_id=recipient_id,
timestamp=datetime_to_timestamp(date_sent),
client=sending_client_name)
obj[TOPIC_NAME] = topic_name
obj['sender_realm_id'] = sender_realm_id
# Render topic_links with the stream's realm instead of the
# user's realm; this is important for messages sent by
# cross-realm bots like NOTIFICATION_BOT.
#
# TODO: We could potentially avoid this database query in
# common cases by optionally passing through the
# stream_realm_id through the code path from do_send_messages
# (where we've already fetched the data). It would involve
# somewhat messy plumbing, but would probably be worth it.
rendering_realm_id = sender_realm_id
if message and recipient_type == Recipient.STREAM:
rendering_realm_id = Stream.objects.get(
id=recipient_type_id).realm_id
obj[TOPIC_LINKS] = bugdown.topic_links(rendering_realm_id, topic_name)
if last_edit_time is not None:
obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
assert edit_history is not None
obj['edit_history'] = ujson.loads(edit_history)
if Message.need_to_render_content(rendered_content,
rendered_content_version,
bugdown.version):
if message is None:
# We really shouldn't be rendering objects in this method, but there is
# a scenario where we upgrade the version of bugdown and fail to run
# management commands to re-render historical messages, and then we
# need to have side effects. This method is optimized to not need full
# blown ORM objects, but the bugdown renderer is unfortunately highly
# coupled to Message, and we also need to persist the new rendered content.
# If we don't have a message object passed in, we get one here. The cost
# of going to the DB here should be overshadowed by the cost of rendering
# and updating the row.
# TODO: see #1379 to eliminate bugdown dependencies
message = Message.objects.select_related().get(id=message_id)
assert message is not None # Hint for mypy.
# It's unfortunate that we need to have side effects on the message
# in some cases.
rendered_content = save_message_rendered_content(message, content)
if rendered_content is not None:
obj['rendered_content'] = rendered_content
else:
obj['rendered_content'] = (
'<p>[Zulip note: Sorry, we could not ' +
'understand the formatting of your message]</p>')
if rendered_content is not None:
obj['is_me_message'] = Message.is_status_message(
content, rendered_content)
else:
obj['is_me_message'] = False
obj['reactions'] = [
ReactionDict.build_dict_from_raw_db_row(reaction)
for reaction in reactions
]
obj['submessages'] = submessages
return obj
def build_message_dict(
message: Optional[Message], message_id: int,
last_edit_time: Optional[datetime.datetime],
edit_history: Optional[str], content: str, subject: str,
pub_date: datetime.datetime, rendered_content: Optional[str],
rendered_content_version: Optional[int], sender_id: int,
sender_realm_id: int, sending_client_name: str, recipient_id: int,
recipient_type: int, recipient_type_id: int,
reactions: List[Dict[str, Any]],
submessages: List[Dict[str, Any]]) -> Dict[str, Any]:
obj = dict(id=message_id,
sender_id=sender_id,
content=content,
recipient_type_id=recipient_type_id,
recipient_type=recipient_type,
recipient_id=recipient_id,
subject=subject,
timestamp=datetime_to_timestamp(pub_date),
client=sending_client_name)
obj['sender_realm_id'] = sender_realm_id
obj['raw_display_recipient'] = get_display_recipient_by_id(
recipient_id, recipient_type, recipient_type_id)
obj['subject_links'] = bugdown.topic_links(sender_realm_id, subject)
if last_edit_time is not None:
obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
assert edit_history is not None
obj['edit_history'] = ujson.loads(edit_history)
if Message.need_to_render_content(rendered_content,
rendered_content_version,
bugdown.version):
if message is None:
# We really shouldn't be rendering objects in this method, but there is
# a scenario where we upgrade the version of bugdown and fail to run
# management commands to re-render historical messages, and then we
# need to have side effects. This method is optimized to not need full
# blown ORM objects, but the bugdown renderer is unfortunately highly
# coupled to Message, and we also need to persist the new rendered content.
# If we don't have a message object passed in, we get one here. The cost
# of going to the DB here should be overshadowed by the cost of rendering
# and updating the row.
# TODO: see #1379 to eliminate bugdown dependencies
message = Message.objects.select_related().get(id=message_id)
assert message is not None # Hint for mypy.
# It's unfortunate that we need to have side effects on the message
# in some cases.
rendered_content = save_message_rendered_content(message, content)
if rendered_content is not None:
obj['rendered_content'] = rendered_content
else:
obj['rendered_content'] = (
'<p>[Zulip note: Sorry, we could not ' +
'understand the formatting of your message]</p>')
if rendered_content is not None:
obj['is_me_message'] = Message.is_status_message(
content, rendered_content)
else:
obj['is_me_message'] = False
obj['reactions'] = [
ReactionDict.build_dict_from_raw_db_row(reaction)
for reaction in reactions
]
obj['submessages'] = submessages
return obj
def build_message_dict(
message_id: int,
last_edit_time: Optional[datetime.datetime],
edit_history: Optional[str],
content: str,
topic_name: str,
date_sent: datetime.datetime,
rendered_content: Optional[str],
rendered_content_version: Optional[int],
sender_id: int,
sender_realm_id: int,
sending_client_name: str,
rendering_realm_id: int,
recipient_id: int,
recipient_type: int,
recipient_type_id: int,
reactions: List[RawReactionRow],
submessages: List[Dict[str, Any]],
) -> Dict[str, Any]:
obj = dict(
id=message_id,
sender_id=sender_id,
content=content,
recipient_type_id=recipient_type_id,
recipient_type=recipient_type,
recipient_id=recipient_id,
timestamp=datetime_to_timestamp(date_sent),
client=sending_client_name,
)
obj[TOPIC_NAME] = topic_name
obj["sender_realm_id"] = sender_realm_id
# Render topic_links with the stream's realm instead of the
# sender's realm; this is important for messages sent by
# cross-realm bots like NOTIFICATION_BOT.
obj[TOPIC_LINKS] = topic_links(rendering_realm_id, topic_name)
if last_edit_time is not None:
obj["last_edit_timestamp"] = datetime_to_timestamp(last_edit_time)
assert edit_history is not None
obj["edit_history"] = orjson.loads(edit_history)
if Message.need_to_render_content(rendered_content,
rendered_content_version,
markdown_version):
# We really shouldn't be rendering objects in this method, but there is
# a scenario where we upgrade the version of Markdown and fail to run
# management commands to re-render historical messages, and then we
# need to have side effects. This method is optimized to not need full
# blown ORM objects, but the Markdown renderer is unfortunately highly
# coupled to Message, and we also need to persist the new rendered content.
# If we don't have a message object passed in, we get one here. The cost
# of going to the DB here should be overshadowed by the cost of rendering
# and updating the row.
# TODO: see #1379 to eliminate Markdown dependencies
message = Message.objects.select_related().get(id=message_id)
assert message is not None # Hint for mypy.
# It's unfortunate that we need to have side effects on the message
# in some cases.
rendered_content = save_message_rendered_content(message, content)
if rendered_content is not None:
obj["rendered_content"] = rendered_content
else:
obj["rendered_content"] = (
"<p>[Zulip note: Sorry, we could not " +
"understand the formatting of your message]</p>")
if rendered_content is not None:
obj["is_me_message"] = Message.is_status_message(
content, rendered_content)
else:
obj["is_me_message"] = False
obj["reactions"] = [
ReactionDict.build_dict_from_raw_db_row(reaction)
for reaction in reactions
]
obj["submessages"] = submessages
return obj
def do_update_message(
user_profile: UserProfile,
target_message: Message,
new_stream: Optional[Stream],
topic_name: Optional[str],
propagate_mode: Optional[str],
send_notification_to_old_thread: bool,
send_notification_to_new_thread: bool,
content: Optional[str],
rendering_result: Optional[MessageRenderingResult],
prior_mention_user_ids: Set[int],
mention_data: Optional[MentionData] = None,
) -> int:
"""
The main function for message editing. A message edit event can
modify:
* the message's content (in which case the caller will have
set both content and rendered_content),
* the topic, in which case the caller will have set topic_name
* or both message's content and the topic
* or stream and/or topic, in which case the caller will have set
new_stream and/or topic_name.
With topic edits, propagate_mode determines whether other message
also have their topics edited.
"""
timestamp = timezone_now()
target_message.last_edit_time = timestamp
event: Dict[str, Any] = {
"type": "update_message",
"user_id": user_profile.id,
"edit_timestamp": datetime_to_timestamp(timestamp),
"message_id": target_message.id,
"rendering_only": False,
}
edit_history_event: EditHistoryEvent = {
"user_id": user_profile.id,
"timestamp": event["edit_timestamp"],
}
changed_messages = [target_message]
realm = user_profile.realm
stream_being_edited = None
if target_message.is_stream_message():
stream_id = target_message.recipient.type_id
stream_being_edited = get_stream_by_id_in_realm(stream_id, realm)
event["stream_name"] = stream_being_edited.name
event["stream_id"] = stream_being_edited.id
ums = UserMessage.objects.filter(message=target_message.id)
if content is not None:
assert rendering_result is not None
# mention_data is required if there's a content edit.
assert mention_data is not None
# add data from group mentions to mentions_user_ids.
for group_id in rendering_result.mentions_user_group_ids:
members = mention_data.get_group_members(group_id)
rendering_result.mentions_user_ids.update(members)
update_user_message_flags(rendering_result, ums)
# One could imagine checking realm.allow_edit_history here and
# modifying the events based on that setting, but doing so
# doesn't really make sense. We need to send the edit event
# to clients regardless, and a client already had access to
# the original/pre-edit content of the message anyway. That
# setting must be enforced on the client side, and making a
# change here simply complicates the logic for clients parsing
# edit history events.
event["orig_content"] = target_message.content
event["orig_rendered_content"] = target_message.rendered_content
edit_history_event["prev_content"] = target_message.content
edit_history_event[
"prev_rendered_content"] = target_message.rendered_content
edit_history_event[
"prev_rendered_content_version"] = target_message.rendered_content_version
target_message.content = content
target_message.rendered_content = rendering_result.rendered_content
target_message.rendered_content_version = markdown_version
event["content"] = content
event["rendered_content"] = rendering_result.rendered_content
event[
"prev_rendered_content_version"] = target_message.rendered_content_version
event["is_me_message"] = Message.is_status_message(
content, rendering_result.rendered_content)
# target_message.has_image and target_message.has_link will have been
# already updated by Markdown rendering in the caller.
target_message.has_attachment = check_attachment_reference_change(
target_message, rendering_result)
if target_message.is_stream_message():
if topic_name is not None:
new_topic_name = topic_name
else:
new_topic_name = target_message.topic_name()
stream_topic: Optional[StreamTopicTarget] = StreamTopicTarget(
stream_id=stream_id,
topic_name=new_topic_name,
)
else:
stream_topic = None
info = get_recipient_info(
realm_id=realm.id,
recipient=target_message.recipient,
sender_id=target_message.sender_id,
stream_topic=stream_topic,
possible_wildcard_mention=mention_data.message_has_wildcards(),
)
event["online_push_user_ids"] = list(info["online_push_user_ids"])
event["pm_mention_push_disabled_user_ids"] = list(
info["pm_mention_push_disabled_user_ids"])
event["pm_mention_email_disabled_user_ids"] = list(
info["pm_mention_email_disabled_user_ids"])
event["stream_push_user_ids"] = list(info["stream_push_user_ids"])
event["stream_email_user_ids"] = list(info["stream_email_user_ids"])
event["muted_sender_user_ids"] = list(info["muted_sender_user_ids"])
event["prior_mention_user_ids"] = list(prior_mention_user_ids)
event["presence_idle_user_ids"] = filter_presence_idle_user_ids(
info["active_user_ids"])
event["all_bot_user_ids"] = list(info["all_bot_user_ids"])
if rendering_result.mentions_wildcard:
event["wildcard_mention_user_ids"] = list(
info["wildcard_mention_user_ids"])
else:
event["wildcard_mention_user_ids"] = []
do_update_mobile_push_notification(
target_message,
prior_mention_user_ids,
rendering_result.mentions_user_ids,
info["stream_push_user_ids"],
)
if topic_name is not None or new_stream is not None:
assert propagate_mode is not None
orig_topic_name = target_message.topic_name()
event["propagate_mode"] = propagate_mode
if new_stream is not None:
assert content is None
assert target_message.is_stream_message()
assert stream_being_edited is not None
edit_history_event["prev_stream"] = stream_being_edited.id
edit_history_event["stream"] = new_stream.id
event[ORIG_TOPIC] = orig_topic_name
assert new_stream.recipient_id is not None
target_message.recipient_id = new_stream.recipient_id
event["new_stream_id"] = new_stream.id
event["propagate_mode"] = propagate_mode
# When messages are moved from one stream to another, some
# users may lose access to those messages, including guest
# users and users not subscribed to the new stream (if it is a
# private stream). For those users, their experience is as
# though the messages were deleted, and we should send a
# delete_message event to them instead.
subs_to_old_stream = get_active_subscriptions_for_stream_id(
stream_id,
include_deactivated_users=True).select_related("user_profile")
subs_to_new_stream = list(
get_active_subscriptions_for_stream_id(
new_stream.id,
include_deactivated_users=True).select_related("user_profile"))
old_stream_sub_ids = [
user.user_profile_id for user in subs_to_old_stream
]
new_stream_sub_ids = [
user.user_profile_id for user in subs_to_new_stream
]
# Get users who aren't subscribed to the new_stream.
subs_losing_usermessages = [
sub for sub in subs_to_old_stream
if sub.user_profile_id not in new_stream_sub_ids
]
# Users who can longer access the message without some action
# from administrators.
subs_losing_access = [
sub for sub in subs_losing_usermessages
if sub.user_profile.is_guest or not new_stream.is_public()
]
ums = ums.exclude(user_profile_id__in=[
sub.user_profile_id for sub in subs_losing_usermessages
])
subs_gaining_usermessages = []
if not new_stream.is_history_public_to_subscribers():
# For private streams, with history not public to subscribers,
# We find out users who are not present in the msgs' old stream
# and create new UserMessage for these users so that they can
# access this message.
subs_gaining_usermessages += [
user_id for user_id in new_stream_sub_ids
if user_id not in old_stream_sub_ids
]
if topic_name is not None:
topic_name = truncate_topic(topic_name)
target_message.set_topic_name(topic_name)
# These fields have legacy field names.
event[ORIG_TOPIC] = orig_topic_name
event[TOPIC_NAME] = topic_name
event[TOPIC_LINKS] = topic_links(target_message.sender.realm_id,
topic_name)
edit_history_event["prev_topic"] = orig_topic_name
edit_history_event["topic"] = topic_name
update_edit_history(target_message, timestamp, edit_history_event)
delete_event_notify_user_ids: List[int] = []
if propagate_mode in ["change_later", "change_all"]:
assert topic_name is not None or new_stream is not None
assert stream_being_edited is not None
# Other messages should only get topic/stream fields in their edit history.
topic_only_edit_history_event: EditHistoryEvent = {
"user_id": edit_history_event["user_id"],
"timestamp": edit_history_event["timestamp"],
}
if topic_name is not None:
topic_only_edit_history_event["prev_topic"] = edit_history_event[
"prev_topic"]
topic_only_edit_history_event["topic"] = edit_history_event[
"topic"]
if new_stream is not None:
topic_only_edit_history_event["prev_stream"] = edit_history_event[
"prev_stream"]
topic_only_edit_history_event["stream"] = edit_history_event[
"stream"]
messages_list = update_messages_for_topic_edit(
acting_user=user_profile,
edited_message=target_message,
propagate_mode=propagate_mode,
orig_topic_name=orig_topic_name,
topic_name=topic_name,
new_stream=new_stream,
old_stream=stream_being_edited,
edit_history_event=topic_only_edit_history_event,
last_edit_time=timestamp,
)
changed_messages += messages_list
if new_stream is not None:
assert stream_being_edited is not None
changed_message_ids = [msg.id for msg in changed_messages]
if subs_gaining_usermessages:
ums_to_create = []
for message_id in changed_message_ids:
for user_profile_id in subs_gaining_usermessages:
# The fact that the user didn't have a UserMessage originally means we can infer that the user
# was not mentioned in the original message (even if mention syntax was present, it would not
# take effect for a user who was not subscribed). If we were editing the message's content, we
# would rerender the message and then use the new stream's data to determine whether this is
# a mention of a subscriber; but as we are not doing so, we choose to preserve the "was this
# mention syntax an actual mention" decision made during the original rendering for implementation
# simplicity. As a result, the only flag to consider applying here is read.
um = UserMessageLite(
user_profile_id=user_profile_id,
message_id=message_id,
flags=UserMessage.flags.read,
)
ums_to_create.append(um)
bulk_insert_ums(ums_to_create)
# Delete UserMessage objects for users who will no
# longer have access to these messages. Note: This could be
# very expensive, since it's N guest users x M messages.
UserMessage.objects.filter(
user_profile_id__in=[
sub.user_profile_id for sub in subs_losing_usermessages
],
message_id__in=changed_message_ids,
).delete()
delete_event: DeleteMessagesEvent = {
"type": "delete_message",
"message_ids": changed_message_ids,
"message_type": "stream",
"stream_id": stream_being_edited.id,
"topic": orig_topic_name,
}
delete_event_notify_user_ids = [
sub.user_profile_id for sub in subs_losing_access
]
send_event(user_profile.realm, delete_event,
delete_event_notify_user_ids)
# Reset the Attachment.is_*_public caches for all messages
# moved to another stream with different access permissions.
if new_stream.invite_only != stream_being_edited.invite_only:
Attachment.objects.filter(
messages__in=changed_message_ids).update(
is_realm_public=None, )
ArchivedAttachment.objects.filter(
messages__in=changed_message_ids).update(
is_realm_public=None, )
if new_stream.is_web_public != stream_being_edited.is_web_public:
Attachment.objects.filter(
messages__in=changed_message_ids).update(
is_web_public=None, )
ArchivedAttachment.objects.filter(
messages__in=changed_message_ids).update(
is_web_public=None, )
# This does message.save(update_fields=[...])
save_message_for_edit_use_case(message=target_message)
realm_id: Optional[int] = None
if stream_being_edited is not None:
realm_id = stream_being_edited.realm_id
event["message_ids"] = update_to_dict_cache(changed_messages, realm_id)
def user_info(um: UserMessage) -> Dict[str, Any]:
return {
"id": um.user_profile_id,
"flags": um.flags_list(),
}
# The following blocks arranges that users who are subscribed to a
# stream and can see history from before they subscribed get
# live-update when old messages are edited (e.g. if the user does
# a topic edit themself).
#
# We still don't send an update event to users who are not
# subscribed to this stream and don't have a UserMessage row. This
# means if a non-subscriber is viewing the narrow, they won't get
# a real-time updates. This is a balance between sending
# message-edit notifications for every public stream to every user
# in the organization (too expansive, and also not what we do for
# newly sent messages anyway) and having magical live-updates
# where possible.
users_to_be_notified = list(map(user_info, ums))
if stream_being_edited is not None:
if stream_being_edited.is_history_public_to_subscribers():
subscriptions = get_active_subscriptions_for_stream_id(
stream_id, include_deactivated_users=False)
# We exclude long-term idle users, since they by
# definition have no active clients.
subscriptions = subscriptions.exclude(
user_profile__long_term_idle=True)
# Remove duplicates by excluding the id of users already
# in users_to_be_notified list. This is the case where a
# user both has a UserMessage row and is a current
# Subscriber
subscriptions = subscriptions.exclude(
user_profile_id__in=[um.user_profile_id for um in ums])
if new_stream is not None:
assert delete_event_notify_user_ids is not None
subscriptions = subscriptions.exclude(
user_profile_id__in=delete_event_notify_user_ids)
# All users that are subscribed to the stream must be
# notified when a message is edited
subscriber_ids = set(
subscriptions.values_list("user_profile_id", flat=True))
if new_stream is not None:
# TODO: Guest users don't see the new moved topic
# unless breadcrumb message for new stream is
# enabled. Excluding these users from receiving this
# event helps us avoid a error traceback for our
# clients. We should figure out a way to inform the
# guest users of this new topic if sending a 'message'
# event for these messages is not an option.
#
# Don't send this event to guest subs who are not
# subscribers of the old stream but are subscribed to
# the new stream; clients will be confused.
old_stream_unsubbed_guests = [
sub for sub in subs_to_new_stream
if sub.user_profile.is_guest
and sub.user_profile_id not in subscriber_ids
]
subscriptions = subscriptions.exclude(user_profile_id__in=[
sub.user_profile_id for sub in old_stream_unsubbed_guests
])
subscriber_ids = set(
subscriptions.values_list("user_profile_id", flat=True))
users_to_be_notified += list(
map(subscriber_info, sorted(list(subscriber_ids))))
# UserTopic updates and the content of notifications depend on
# whether we've moved the entire topic, or just part of it. We
# make that determination here.
moved_all_visible_messages = False
if topic_name is not None or new_stream is not None:
assert stream_being_edited is not None
if propagate_mode == "change_all":
moved_all_visible_messages = True
else:
# With other propagate modes, if the user in fact moved
# all messages in the stream, we want to explain it was a
# full-topic move.
#
# For security model reasons, we don't want to allow a
# user to take any action that would leak information
# about older messages they cannot access (E.g. the only
# remaining messages are in a stream without shared
# history). The bulk_access_messages call below addresses
# that concern.
#
# bulk_access_messages is inefficient for this task, since
# we just want to do the exists() version of this
# query. But it's nice to reuse code, and this bulk
# operation is likely cheaper than a `GET /messages`
# unless the topic has thousands of messages of history.
assert stream_being_edited.recipient_id is not None
unmoved_messages = messages_for_topic(
stream_being_edited.recipient_id,
orig_topic_name,
)
visible_unmoved_messages = bulk_access_messages(
user_profile, unmoved_messages, stream=stream_being_edited)
moved_all_visible_messages = len(visible_unmoved_messages) == 0
# Migrate muted topic configuration in the following circumstances:
#
# * If propagate_mode is change_all, do so unconditionally.
#
# * If propagate_mode is change_later or change_one, do so when
# the acting user has moved the entire topic (as visible to them).
#
# This rule corresponds to checking moved_all_visible_messages.
#
# We may want more complex behavior in cases where one appears to
# be merging topics (E.g. there are existing messages in the
# target topic).
if moved_all_visible_messages:
assert stream_being_edited is not None
assert topic_name is not None or new_stream is not None
for muting_user in get_users_muting_topic(stream_being_edited.id,
orig_topic_name):
# TODO: Ideally, this would be a bulk update operation,
# because we are doing database operations in a loop here.
#
# This loop is only acceptable in production because it is
# rare for more than a few users to have muted an
# individual topic that is being moved; as of this
# writing, no individual topic in Zulip Cloud had been
# muted by more than 100 users.
if new_stream is not None and muting_user.id in delete_event_notify_user_ids:
# If the messages are being moved to a stream the user
# cannot access, then we treat this as the
# messages/topic being deleted for this user. This is
# important for security reasons; we don't want to
# give users a UserTopic row in a stream they cannot
# access. Unmute the topic for such users.
do_unmute_topic(muting_user, stream_being_edited,
orig_topic_name)
else:
# Otherwise, we move the muted topic record for the user.
# We call remove_topic_mute rather than do_unmute_topic to
# avoid sending two events with new muted topics in
# immediate succession; this is correct only because
# muted_topics events always send the full set of topics.
remove_topic_mute(muting_user, stream_being_edited.id,
orig_topic_name)
do_mute_topic(
muting_user,
new_stream
if new_stream is not None else stream_being_edited,
topic_name if topic_name is not None else orig_topic_name,
ignore_duplicate=True,
)
send_event(user_profile.realm, event, users_to_be_notified)
if len(
changed_messages
) > 0 and new_stream is not None and stream_being_edited is not None:
# Notify users that the topic was moved.
changed_messages_count = len(changed_messages)
old_thread_notification_string = None
if send_notification_to_old_thread:
if moved_all_visible_messages:
old_thread_notification_string = gettext_lazy(
"This topic was moved to {new_location} by {user}.")
elif changed_messages_count == 1:
old_thread_notification_string = gettext_lazy(
"A message was moved from this topic to {new_location} by {user}."
)
else:
old_thread_notification_string = gettext_lazy(
"{changed_messages_count} messages were moved from this topic to {new_location} by {user}."
)
new_thread_notification_string = None
if send_notification_to_new_thread:
if moved_all_visible_messages:
new_thread_notification_string = gettext_lazy(
"This topic was moved here from {old_location} by {user}.")
elif changed_messages_count == 1:
new_thread_notification_string = gettext_lazy(
"A message was moved here from {old_location} by {user}.")
else:
new_thread_notification_string = gettext_lazy(
"{changed_messages_count} messages were moved here from {old_location} by {user}."
)
send_message_moved_breadcrumbs(
user_profile,
stream_being_edited,
orig_topic_name,
old_thread_notification_string,
new_stream,
topic_name,
new_thread_notification_string,
changed_messages_count,
)
if (topic_name is not None and new_stream is None and content is None
and len(changed_messages) > 0):
assert stream_being_edited is not None
maybe_send_resolve_topic_notifications(
user_profile=user_profile,
stream=stream_being_edited,
old_topic=orig_topic_name,
new_topic=topic_name,
changed_messages=changed_messages,
)
return len(changed_messages)
def build_message_dict(
message: Optional[Message],
message_id: int,
last_edit_time: Optional[datetime.datetime],
edit_history: Optional[str],
content: str,
topic_name: str,
pub_date: datetime.datetime,
rendered_content: Optional[str],
rendered_content_version: Optional[int],
sender_id: int,
sender_realm_id: int,
sending_client_name: str,
recipient_id: int,
recipient_type: int,
recipient_type_id: int,
reactions: List[Dict[str, Any]],
submessages: List[Dict[str, Any]]
) -> Dict[str, Any]:
obj = dict(
id = message_id,
sender_id = sender_id,
content = content,
recipient_type_id = recipient_type_id,
recipient_type = recipient_type,
recipient_id = recipient_id,
timestamp = datetime_to_timestamp(pub_date),
client = sending_client_name)
obj[TOPIC_NAME] = topic_name
obj['sender_realm_id'] = sender_realm_id
obj['raw_display_recipient'] = get_display_recipient_by_id(
recipient_id,
recipient_type,
recipient_type_id
)
obj[TOPIC_LINKS] = bugdown.topic_links(sender_realm_id, topic_name)
if last_edit_time is not None:
obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
assert edit_history is not None
obj['edit_history'] = ujson.loads(edit_history)
if Message.need_to_render_content(rendered_content, rendered_content_version, bugdown.version):
if message is None:
# We really shouldn't be rendering objects in this method, but there is
# a scenario where we upgrade the version of bugdown and fail to run
# management commands to re-render historical messages, and then we
# need to have side effects. This method is optimized to not need full
# blown ORM objects, but the bugdown renderer is unfortunately highly
# coupled to Message, and we also need to persist the new rendered content.
# If we don't have a message object passed in, we get one here. The cost
# of going to the DB here should be overshadowed by the cost of rendering
# and updating the row.
# TODO: see #1379 to eliminate bugdown dependencies
message = Message.objects.select_related().get(id=message_id)
assert message is not None # Hint for mypy.
# It's unfortunate that we need to have side effects on the message
# in some cases.
rendered_content = save_message_rendered_content(message, content)
if rendered_content is not None:
obj['rendered_content'] = rendered_content
else:
obj['rendered_content'] = ('<p>[Zulip note: Sorry, we could not ' +
'understand the formatting of your message]</p>')
if rendered_content is not None:
obj['is_me_message'] = Message.is_status_message(content, rendered_content)
else:
obj['is_me_message'] = False
obj['reactions'] = [ReactionDict.build_dict_from_raw_db_row(reaction)
for reaction in reactions]
obj['submessages'] = submessages
return obj
def build_message_dict(
apply_markdown,
message,
message_id,
last_edit_time,
edit_history,
content,
subject,
pub_date,
rendered_content,
rendered_content_version,
sender_id,
sender_email,
sender_realm_id,
sender_realm_str,
sender_avatar_source,
sender_avatar_version,
sender_is_mirror_dummy,
sending_client_name,
recipient_id,
recipient_type,
recipient_type_id,
reactions
):
# type: (bool, Optional[Message], int, Optional[datetime.datetime], Optional[Text], Text, Text, datetime.datetime, Optional[Text], Optional[int], int, Text, int, Text, Text, int, bool, Text, int, int, int, List[Dict[str, Any]]) -> Dict[str, Any]
avatar_url = avatar_url_from_dict(dict(
avatar_source=sender_avatar_source,
avatar_version=sender_avatar_version,
email=sender_email,
id=sender_id,
realm_id=sender_realm_id))
obj = dict(
id = message_id,
sender_email = sender_email,
sender_realm_str = sender_realm_str,
sender_id = sender_id,
recipient_type_id = recipient_type_id,
recipient_type = recipient_type,
recipient_id = recipient_id,
subject = subject,
timestamp = datetime_to_timestamp(pub_date),
avatar_url = avatar_url,
client = sending_client_name)
obj['raw_display_recipient'] = get_display_recipient_by_id(
recipient_id,
recipient_type,
recipient_type_id
)
obj['sender_is_mirror_dummy'] = sender_is_mirror_dummy
obj['subject_links'] = bugdown.subject_links(sender_realm_id, subject)
if last_edit_time is not None:
obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
assert edit_history is not None
obj['edit_history'] = ujson.loads(edit_history)
if apply_markdown:
if Message.need_to_render_content(rendered_content, rendered_content_version, bugdown.version):
if message is None:
# We really shouldn't be rendering objects in this method, but there is
# a scenario where we upgrade the version of bugdown and fail to run
# management commands to re-render historical messages, and then we
# need to have side effects. This method is optimized to not need full
# blown ORM objects, but the bugdown renderer is unfortunately highly
# coupled to Message, and we also need to persist the new rendered content.
# If we don't have a message object passed in, we get one here. The cost
# of going to the DB here should be overshadowed by the cost of rendering
# and updating the row.
# TODO: see #1379 to eliminate bugdown dependencies
message = Message.objects.select_related().get(id=message_id)
assert message is not None # Hint for mypy.
# It's unfortunate that we need to have side effects on the message
# in some cases.
rendered_content = render_markdown(message, content, realm=message.get_realm())
message.rendered_content = rendered_content
message.rendered_content_version = bugdown.version
message.save_rendered_content()
if rendered_content is not None:
obj['content'] = rendered_content
else:
obj['content'] = u'<p>[Zulip note: Sorry, we could not understand the formatting of your message]</p>'
obj['content_type'] = 'text/html'
else:
obj['content'] = content
obj['content_type'] = 'text/x-markdown'
if rendered_content is not None:
obj['is_me_message'] = Message.is_status_message(content, rendered_content)
else:
obj['is_me_message'] = False
obj['reactions'] = [ReactionDict.build_dict_from_raw_db_row(reaction)
for reaction in reactions]
return obj
