def render_markdown(message, content, realm=None, realm_alert_words=None, message_users=None): # type: (Message, Text, Optional[Realm], Optional[RealmAlertWords], Set[UserProfile]) -> Text """Return HTML for given markdown. Bugdown may add properties to the message object such as `mentions_user_ids` and `mentions_wildcard`. These are only on this Django object and are not saved in the database. """ if message_users is None: message_user_ids = set() # type: Set[int] else: message_user_ids = {u.id for u in message_users} if message is not None: message.mentions_wildcard = False message.is_me_message = False message.mentions_user_ids = set() message.alert_words = set() message.links_for_preview = set() if realm is None: realm = message.get_realm() possible_words = set() # type: Set[Text] if realm_alert_words is not None: for user_id, words in realm_alert_words.items(): if user_id in message_user_ids: possible_words.update(set(words)) if message is None: # If we don't have a message, then we are in the compose preview # codepath, so we know we are dealing with a human. sent_by_bot = False else: sent_by_bot = get_user_profile_by_id(message.sender_id).is_bot # DO MAIN WORK HERE -- call bugdown to convert rendered_content = bugdown.convert(content, message=message, message_realm=realm, possible_words=possible_words, sent_by_bot=sent_by_bot) if message is not None: message.user_ids_with_alert_words = set() if realm_alert_words is not None: for user_id, words in realm_alert_words.items(): if user_id in message_user_ids: if set(words).intersection(message.alert_words): message.user_ids_with_alert_words.add(user_id) message.is_me_message = Message.is_status_message( content, rendered_content) return rendered_content
def archive(request: HttpRequest, stream_id: int, topic_name: str) -> HttpResponse: def get_response( rendered_message_list: List[str], is_web_public: bool, stream_name: str ) -> HttpResponse: return render( request, "zerver/archive/index.html", context={ "is_web_public": is_web_public, "message_list": rendered_message_list, "stream": stream_name, "topic": topic_name, }, ) try: stream = access_web_public_stream(stream_id, request.realm) except JsonableError: return get_response([], False, "") all_messages = list( messages_for_topic( stream_recipient_id=stream.recipient_id, topic_name=topic_name, ) .select_related("sender") .order_by("date_sent"), ) if not all_messages: return get_response([], True, stream.name) rendered_message_list = [] prev_sender: Optional[UserProfile] = None for msg in all_messages: include_sender = False status_message = Message.is_status_message(msg.content, msg.rendered_content) if not prev_sender or prev_sender != msg.sender or status_message: if status_message: prev_sender = None else: prev_sender = msg.sender include_sender = True if status_message: status_message = msg.rendered_content[4 + 3 : -4] context = { "sender_full_name": msg.sender.full_name, "timestampstr": datetime_to_timestamp( msg.last_edit_time if msg.last_edit_time else msg.date_sent ), "message_content": msg.rendered_content, "avatar_url": get_gravatar_url(msg.sender.delivery_email, 1), "include_sender": include_sender, "status_message": status_message, } rendered_msg = loader.render_to_string("zerver/archive/single_message.html", context) rendered_message_list.append(rendered_msg) return get_response(rendered_message_list, True, stream.name)
def archive(request: HttpRequest, stream_id: int, topic_name: str) -> HttpResponse: def get_response(rendered_message_list: List[str], is_web_public: bool, stream_name: str) -> HttpResponse: return render( request, 'zerver/archive/index.html', context={ 'is_web_public': is_web_public, 'message_list': rendered_message_list, 'stream': stream_name, 'topic': topic_name, } ) try: stream = get_stream_by_id(stream_id) except JsonableError: return get_response([], False, '') if not stream.is_web_public: return get_response([], False, '') all_messages = list(Message.objects.select_related( 'sender').filter(recipient__type_id=stream_id, subject=topic_name).order_by('pub_date')) if not all_messages: return get_response([], True, stream.name) rendered_message_list = [] prev_sender = None for msg in all_messages: include_sender = False status_message = Message.is_status_message(msg.content, msg.rendered_content) if not prev_sender or prev_sender != msg.sender or status_message: if status_message: prev_sender = None else: prev_sender = msg.sender include_sender = True if status_message: status_message = msg.rendered_content[4+3: -4] context = { 'sender_full_name': msg.sender.full_name, 'timestampstr': datetime_to_timestamp(msg.last_edit_time if msg.last_edit_time else msg.pub_date), 'message_content': msg.rendered_content, 'avatar_url': get_gravatar_url(msg.sender.email, 1), 'include_sender': include_sender, 'status_message': status_message, } rendered_msg = loader.render_to_string('zerver/archive/single_message.html', context) rendered_message_list.append(rendered_msg) return get_response(rendered_message_list, True, stream.name)
def archive(request: HttpRequest, stream_id: int, topic_name: str) -> HttpResponse: def get_response(rendered_message_list: List[str], is_web_public: bool, stream_name: str) -> HttpResponse: return render( request, 'zerver/archive/index.html', context={ 'is_web_public': is_web_public, 'message_list': rendered_message_list, 'stream': stream_name, 'topic': topic_name, } ) try: stream = get_stream_by_id(stream_id) except JsonableError: return get_response([], False, '') if not stream.is_web_public: return get_response([], False, '') all_messages = list(Message.objects.select_related( 'sender').filter(recipient__type_id=stream_id, subject=topic_name).order_by('pub_date')) if not all_messages: return get_response([], True, stream.name) rendered_message_list = [] prev_sender = None for msg in all_messages: include_sender = False status_message = Message.is_status_message(msg.content, msg.rendered_content) if not prev_sender or prev_sender != msg.sender or status_message: if status_message: prev_sender = None else: prev_sender = msg.sender include_sender = True if status_message: status_message = msg.rendered_content[4+3: -4] context = { 'sender_full_name': msg.sender.full_name, 'timestampstr': datetime_to_timestamp(msg.last_edit_time if msg.last_edit_time else msg.pub_date), 'message_content': msg.rendered_content, 'avatar_url': get_gravatar_url(msg.sender.email, 1), 'include_sender': include_sender, 'status_message': status_message, } rendered_msg = loader.render_to_string('zerver/archive/single_message.html', context) rendered_message_list.append(rendered_msg) return get_response(rendered_message_list, True, stream.name)
def render_markdown(message, content, realm_id=None, realm_alert_words=None, message_users=None): # type: (Message, Text, Optional[int], Optional[RealmAlertWords], Set[UserProfile]) -> Text """Return HTML for given markdown. Bugdown may add properties to the message object such as `mentions_user_ids` and `mentions_wildcard`. These are only on this Django object and are not saved in the database. """ if message_users is None: message_user_ids = set() # type: Set[int] else: message_user_ids = {u.id for u in message_users} if message is not None: message.mentions_wildcard = False message.is_me_message = False message.mentions_user_ids = set() message.alert_words = set() message.links_for_preview = set() if realm_id is None: realm_id = message.sender.realm_id if message.sending_client.name == "zephyr_mirror" and message.sender.realm.is_zephyr_mirror_realm: # Use slightly customized Markdown processor for content # delivered via zephyr_mirror realm_id = bugdown.ZEPHYR_MIRROR_BUGDOWN_KEY possible_words = set() # type: Set[Text] if realm_alert_words is not None: for user_id, words in realm_alert_words.items(): if user_id in message_user_ids: possible_words.update(set(words)) # DO MAIN WORK HERE -- call bugdown to convert rendered_content = bugdown.convert(content, realm_filters_key=realm_id, message=message, possible_words=possible_words) if message is not None: message.user_ids_with_alert_words = set() if realm_alert_words is not None: for user_id, words in realm_alert_words.items(): if user_id in message_user_ids: if set(words).intersection(message.alert_words): message.user_ids_with_alert_words.add(user_id) message.is_me_message = Message.is_status_message( content, rendered_content) return rendered_content
def render_markdown(message, content, realm=None, realm_alert_words=None, message_users=None): # type: (Message, Text, Optional[Realm], Optional[RealmAlertWords], Set[UserProfile]) -> Text """Return HTML for given markdown. Bugdown may add properties to the message object such as `mentions_user_ids` and `mentions_wildcard`. These are only on this Django object and are not saved in the database. """ if message_users is None: message_user_ids = set() # type: Set[int] else: message_user_ids = {u.id for u in message_users} if message is not None: message.mentions_wildcard = False message.is_me_message = False message.mentions_user_ids = set() message.alert_words = set() message.links_for_preview = set() message.outgoing_webhook_bot_triggers = [] if realm is None: realm = message.get_realm() possible_words = set() # type: Set[Text] if realm_alert_words is not None: for user_id, words in realm_alert_words.items(): if user_id in message_user_ids: possible_words.update(set(words)) if message is None: # If we don't have a message, then we are in the compose preview # codepath, so we know we are dealing with a human. sent_by_bot = False else: sent_by_bot = get_user_profile_by_id(message.sender_id).is_bot # DO MAIN WORK HERE -- call bugdown to convert rendered_content = bugdown.convert(content, message=message, message_realm=realm, possible_words=possible_words, sent_by_bot=sent_by_bot) if message is not None: message.user_ids_with_alert_words = set() if realm_alert_words is not None: for user_id, words in realm_alert_words.items(): if user_id in message_user_ids: if set(words).intersection(message.alert_words): message.user_ids_with_alert_words.add(user_id) message.is_me_message = Message.is_status_message(content, rendered_content) return rendered_content
def render_markdown(message, content, realm_id=None, realm_alert_words=None, message_users=None): # type: (Message, Text, Optional[int], Optional[RealmAlertWords], Set[UserProfile]) -> Text """Return HTML for given markdown. Bugdown may add properties to the message object such as `mentions_user_ids` and `mentions_wildcard`. These are only on this Django object and are not saved in the database. """ if message_users is None: message_user_ids = set() # type: Set[int] else: message_user_ids = {u.id for u in message_users} if message is not None: message.mentions_wildcard = False message.is_me_message = False message.mentions_user_ids = set() message.alert_words = set() message.links_for_preview = set() if realm_id is None: realm_id = message.sender.realm_id if message.sending_client.name == "zephyr_mirror" and message.sender.realm.is_zephyr_mirror_realm: # Use slightly customized Markdown processor for content # delivered via zephyr_mirror realm_id = bugdown.ZEPHYR_MIRROR_BUGDOWN_KEY possible_words = set() # type: Set[Text] if realm_alert_words is not None: for user_id, words in realm_alert_words.items(): if user_id in message_user_ids: possible_words.update(set(words)) # DO MAIN WORK HERE -- call bugdown to convert rendered_content = bugdown.convert(content, realm_filters_key=realm_id, message=message, possible_words=possible_words) if message is not None: message.user_ids_with_alert_words = set() if realm_alert_words is not None: for user_id, words in realm_alert_words.items(): if user_id in message_user_ids: if set(words).intersection(message.alert_words): message.user_ids_with_alert_words.add(user_id) message.is_me_message = Message.is_status_message(content, rendered_content) return rendered_content
def build_message_dict( message: Optional[Message], message_id: int, last_edit_time: Optional[datetime.datetime], edit_history: Optional[str], content: str, topic_name: str, date_sent: datetime.datetime, rendered_content: Optional[str], rendered_content_version: Optional[int], sender_id: int, sender_realm_id: int, sending_client_name: str, recipient_id: int, recipient_type: int, recipient_type_id: int, reactions: List[Dict[str, Any]], submessages: List[Dict[str, Any]]) -> Dict[str, Any]: obj = dict(id=message_id, sender_id=sender_id, content=content, recipient_type_id=recipient_type_id, recipient_type=recipient_type, recipient_id=recipient_id, timestamp=datetime_to_timestamp(date_sent), client=sending_client_name) obj[TOPIC_NAME] = topic_name obj['sender_realm_id'] = sender_realm_id # Render topic_links with the stream's realm instead of the # user's realm; this is important for messages sent by # cross-realm bots like NOTIFICATION_BOT. # # TODO: We could potentially avoid this database query in # common cases by optionally passing through the # stream_realm_id through the code path from do_send_messages # (where we've already fetched the data). It would involve # somewhat messy plumbing, but would probably be worth it. rendering_realm_id = sender_realm_id if message and recipient_type == Recipient.STREAM: rendering_realm_id = Stream.objects.get( id=recipient_type_id).realm_id obj[TOPIC_LINKS] = bugdown.topic_links(rendering_realm_id, topic_name) if last_edit_time is not None: obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time) assert edit_history is not None obj['edit_history'] = ujson.loads(edit_history) if Message.need_to_render_content(rendered_content, rendered_content_version, bugdown.version): if message is None: # We really shouldn't be rendering objects in this method, but there is # a scenario where we upgrade the version of bugdown and fail to run # management commands to re-render historical messages, and then we # need to have side effects. This method is optimized to not need full # blown ORM objects, but the bugdown renderer is unfortunately highly # coupled to Message, and we also need to persist the new rendered content. # If we don't have a message object passed in, we get one here. The cost # of going to the DB here should be overshadowed by the cost of rendering # and updating the row. # TODO: see #1379 to eliminate bugdown dependencies message = Message.objects.select_related().get(id=message_id) assert message is not None # Hint for mypy. # It's unfortunate that we need to have side effects on the message # in some cases. rendered_content = save_message_rendered_content(message, content) if rendered_content is not None: obj['rendered_content'] = rendered_content else: obj['rendered_content'] = ( '<p>[Zulip note: Sorry, we could not ' + 'understand the formatting of your message]</p>') if rendered_content is not None: obj['is_me_message'] = Message.is_status_message( content, rendered_content) else: obj['is_me_message'] = False obj['reactions'] = [ ReactionDict.build_dict_from_raw_db_row(reaction) for reaction in reactions ] obj['submessages'] = submessages return obj
def build_message_dict( message: Optional[Message], message_id: int, last_edit_time: Optional[datetime.datetime], edit_history: Optional[str], content: str, subject: str, pub_date: datetime.datetime, rendered_content: Optional[str], rendered_content_version: Optional[int], sender_id: int, sender_realm_id: int, sending_client_name: str, recipient_id: int, recipient_type: int, recipient_type_id: int, reactions: List[Dict[str, Any]], submessages: List[Dict[str, Any]]) -> Dict[str, Any]: obj = dict(id=message_id, sender_id=sender_id, content=content, recipient_type_id=recipient_type_id, recipient_type=recipient_type, recipient_id=recipient_id, subject=subject, timestamp=datetime_to_timestamp(pub_date), client=sending_client_name) obj['sender_realm_id'] = sender_realm_id obj['raw_display_recipient'] = get_display_recipient_by_id( recipient_id, recipient_type, recipient_type_id) obj['subject_links'] = bugdown.topic_links(sender_realm_id, subject) if last_edit_time is not None: obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time) assert edit_history is not None obj['edit_history'] = ujson.loads(edit_history) if Message.need_to_render_content(rendered_content, rendered_content_version, bugdown.version): if message is None: # We really shouldn't be rendering objects in this method, but there is # a scenario where we upgrade the version of bugdown and fail to run # management commands to re-render historical messages, and then we # need to have side effects. This method is optimized to not need full # blown ORM objects, but the bugdown renderer is unfortunately highly # coupled to Message, and we also need to persist the new rendered content. # If we don't have a message object passed in, we get one here. The cost # of going to the DB here should be overshadowed by the cost of rendering # and updating the row. # TODO: see #1379 to eliminate bugdown dependencies message = Message.objects.select_related().get(id=message_id) assert message is not None # Hint for mypy. # It's unfortunate that we need to have side effects on the message # in some cases. rendered_content = save_message_rendered_content(message, content) if rendered_content is not None: obj['rendered_content'] = rendered_content else: obj['rendered_content'] = ( '<p>[Zulip note: Sorry, we could not ' + 'understand the formatting of your message]</p>') if rendered_content is not None: obj['is_me_message'] = Message.is_status_message( content, rendered_content) else: obj['is_me_message'] = False obj['reactions'] = [ ReactionDict.build_dict_from_raw_db_row(reaction) for reaction in reactions ] obj['submessages'] = submessages return obj
def build_message_dict( message_id: int, last_edit_time: Optional[datetime.datetime], edit_history: Optional[str], content: str, topic_name: str, date_sent: datetime.datetime, rendered_content: Optional[str], rendered_content_version: Optional[int], sender_id: int, sender_realm_id: int, sending_client_name: str, rendering_realm_id: int, recipient_id: int, recipient_type: int, recipient_type_id: int, reactions: List[RawReactionRow], submessages: List[Dict[str, Any]], ) -> Dict[str, Any]: obj = dict( id=message_id, sender_id=sender_id, content=content, recipient_type_id=recipient_type_id, recipient_type=recipient_type, recipient_id=recipient_id, timestamp=datetime_to_timestamp(date_sent), client=sending_client_name, ) obj[TOPIC_NAME] = topic_name obj["sender_realm_id"] = sender_realm_id # Render topic_links with the stream's realm instead of the # sender's realm; this is important for messages sent by # cross-realm bots like NOTIFICATION_BOT. obj[TOPIC_LINKS] = topic_links(rendering_realm_id, topic_name) if last_edit_time is not None: obj["last_edit_timestamp"] = datetime_to_timestamp(last_edit_time) assert edit_history is not None obj["edit_history"] = orjson.loads(edit_history) if Message.need_to_render_content(rendered_content, rendered_content_version, markdown_version): # We really shouldn't be rendering objects in this method, but there is # a scenario where we upgrade the version of Markdown and fail to run # management commands to re-render historical messages, and then we # need to have side effects. This method is optimized to not need full # blown ORM objects, but the Markdown renderer is unfortunately highly # coupled to Message, and we also need to persist the new rendered content. # If we don't have a message object passed in, we get one here. The cost # of going to the DB here should be overshadowed by the cost of rendering # and updating the row. # TODO: see #1379 to eliminate Markdown dependencies message = Message.objects.select_related().get(id=message_id) assert message is not None # Hint for mypy. # It's unfortunate that we need to have side effects on the message # in some cases. rendered_content = save_message_rendered_content(message, content) if rendered_content is not None: obj["rendered_content"] = rendered_content else: obj["rendered_content"] = ( "<p>[Zulip note: Sorry, we could not " + "understand the formatting of your message]</p>") if rendered_content is not None: obj["is_me_message"] = Message.is_status_message( content, rendered_content) else: obj["is_me_message"] = False obj["reactions"] = [ ReactionDict.build_dict_from_raw_db_row(reaction) for reaction in reactions ] obj["submessages"] = submessages return obj
def do_update_message( user_profile: UserProfile, target_message: Message, new_stream: Optional[Stream], topic_name: Optional[str], propagate_mode: Optional[str], send_notification_to_old_thread: bool, send_notification_to_new_thread: bool, content: Optional[str], rendering_result: Optional[MessageRenderingResult], prior_mention_user_ids: Set[int], mention_data: Optional[MentionData] = None, ) -> int: """ The main function for message editing. A message edit event can modify: * the message's content (in which case the caller will have set both content and rendered_content), * the topic, in which case the caller will have set topic_name * or both message's content and the topic * or stream and/or topic, in which case the caller will have set new_stream and/or topic_name. With topic edits, propagate_mode determines whether other message also have their topics edited. """ timestamp = timezone_now() target_message.last_edit_time = timestamp event: Dict[str, Any] = { "type": "update_message", "user_id": user_profile.id, "edit_timestamp": datetime_to_timestamp(timestamp), "message_id": target_message.id, "rendering_only": False, } edit_history_event: EditHistoryEvent = { "user_id": user_profile.id, "timestamp": event["edit_timestamp"], } changed_messages = [target_message] realm = user_profile.realm stream_being_edited = None if target_message.is_stream_message(): stream_id = target_message.recipient.type_id stream_being_edited = get_stream_by_id_in_realm(stream_id, realm) event["stream_name"] = stream_being_edited.name event["stream_id"] = stream_being_edited.id ums = UserMessage.objects.filter(message=target_message.id) if content is not None: assert rendering_result is not None # mention_data is required if there's a content edit. assert mention_data is not None # add data from group mentions to mentions_user_ids. for group_id in rendering_result.mentions_user_group_ids: members = mention_data.get_group_members(group_id) rendering_result.mentions_user_ids.update(members) update_user_message_flags(rendering_result, ums) # One could imagine checking realm.allow_edit_history here and # modifying the events based on that setting, but doing so # doesn't really make sense. We need to send the edit event # to clients regardless, and a client already had access to # the original/pre-edit content of the message anyway. That # setting must be enforced on the client side, and making a # change here simply complicates the logic for clients parsing # edit history events. event["orig_content"] = target_message.content event["orig_rendered_content"] = target_message.rendered_content edit_history_event["prev_content"] = target_message.content edit_history_event[ "prev_rendered_content"] = target_message.rendered_content edit_history_event[ "prev_rendered_content_version"] = target_message.rendered_content_version target_message.content = content target_message.rendered_content = rendering_result.rendered_content target_message.rendered_content_version = markdown_version event["content"] = content event["rendered_content"] = rendering_result.rendered_content event[ "prev_rendered_content_version"] = target_message.rendered_content_version event["is_me_message"] = Message.is_status_message( content, rendering_result.rendered_content) # target_message.has_image and target_message.has_link will have been # already updated by Markdown rendering in the caller. target_message.has_attachment = check_attachment_reference_change( target_message, rendering_result) if target_message.is_stream_message(): if topic_name is not None: new_topic_name = topic_name else: new_topic_name = target_message.topic_name() stream_topic: Optional[StreamTopicTarget] = StreamTopicTarget( stream_id=stream_id, topic_name=new_topic_name, ) else: stream_topic = None info = get_recipient_info( realm_id=realm.id, recipient=target_message.recipient, sender_id=target_message.sender_id, stream_topic=stream_topic, possible_wildcard_mention=mention_data.message_has_wildcards(), ) event["online_push_user_ids"] = list(info["online_push_user_ids"]) event["pm_mention_push_disabled_user_ids"] = list( info["pm_mention_push_disabled_user_ids"]) event["pm_mention_email_disabled_user_ids"] = list( info["pm_mention_email_disabled_user_ids"]) event["stream_push_user_ids"] = list(info["stream_push_user_ids"]) event["stream_email_user_ids"] = list(info["stream_email_user_ids"]) event["muted_sender_user_ids"] = list(info["muted_sender_user_ids"]) event["prior_mention_user_ids"] = list(prior_mention_user_ids) event["presence_idle_user_ids"] = filter_presence_idle_user_ids( info["active_user_ids"]) event["all_bot_user_ids"] = list(info["all_bot_user_ids"]) if rendering_result.mentions_wildcard: event["wildcard_mention_user_ids"] = list( info["wildcard_mention_user_ids"]) else: event["wildcard_mention_user_ids"] = [] do_update_mobile_push_notification( target_message, prior_mention_user_ids, rendering_result.mentions_user_ids, info["stream_push_user_ids"], ) if topic_name is not None or new_stream is not None: assert propagate_mode is not None orig_topic_name = target_message.topic_name() event["propagate_mode"] = propagate_mode if new_stream is not None: assert content is None assert target_message.is_stream_message() assert stream_being_edited is not None edit_history_event["prev_stream"] = stream_being_edited.id edit_history_event["stream"] = new_stream.id event[ORIG_TOPIC] = orig_topic_name assert new_stream.recipient_id is not None target_message.recipient_id = new_stream.recipient_id event["new_stream_id"] = new_stream.id event["propagate_mode"] = propagate_mode # When messages are moved from one stream to another, some # users may lose access to those messages, including guest # users and users not subscribed to the new stream (if it is a # private stream). For those users, their experience is as # though the messages were deleted, and we should send a # delete_message event to them instead. subs_to_old_stream = get_active_subscriptions_for_stream_id( stream_id, include_deactivated_users=True).select_related("user_profile") subs_to_new_stream = list( get_active_subscriptions_for_stream_id( new_stream.id, include_deactivated_users=True).select_related("user_profile")) old_stream_sub_ids = [ user.user_profile_id for user in subs_to_old_stream ] new_stream_sub_ids = [ user.user_profile_id for user in subs_to_new_stream ] # Get users who aren't subscribed to the new_stream. subs_losing_usermessages = [ sub for sub in subs_to_old_stream if sub.user_profile_id not in new_stream_sub_ids ] # Users who can longer access the message without some action # from administrators. subs_losing_access = [ sub for sub in subs_losing_usermessages if sub.user_profile.is_guest or not new_stream.is_public() ] ums = ums.exclude(user_profile_id__in=[ sub.user_profile_id for sub in subs_losing_usermessages ]) subs_gaining_usermessages = [] if not new_stream.is_history_public_to_subscribers(): # For private streams, with history not public to subscribers, # We find out users who are not present in the msgs' old stream # and create new UserMessage for these users so that they can # access this message. subs_gaining_usermessages += [ user_id for user_id in new_stream_sub_ids if user_id not in old_stream_sub_ids ] if topic_name is not None: topic_name = truncate_topic(topic_name) target_message.set_topic_name(topic_name) # These fields have legacy field names. event[ORIG_TOPIC] = orig_topic_name event[TOPIC_NAME] = topic_name event[TOPIC_LINKS] = topic_links(target_message.sender.realm_id, topic_name) edit_history_event["prev_topic"] = orig_topic_name edit_history_event["topic"] = topic_name update_edit_history(target_message, timestamp, edit_history_event) delete_event_notify_user_ids: List[int] = [] if propagate_mode in ["change_later", "change_all"]: assert topic_name is not None or new_stream is not None assert stream_being_edited is not None # Other messages should only get topic/stream fields in their edit history. topic_only_edit_history_event: EditHistoryEvent = { "user_id": edit_history_event["user_id"], "timestamp": edit_history_event["timestamp"], } if topic_name is not None: topic_only_edit_history_event["prev_topic"] = edit_history_event[ "prev_topic"] topic_only_edit_history_event["topic"] = edit_history_event[ "topic"] if new_stream is not None: topic_only_edit_history_event["prev_stream"] = edit_history_event[ "prev_stream"] topic_only_edit_history_event["stream"] = edit_history_event[ "stream"] messages_list = update_messages_for_topic_edit( acting_user=user_profile, edited_message=target_message, propagate_mode=propagate_mode, orig_topic_name=orig_topic_name, topic_name=topic_name, new_stream=new_stream, old_stream=stream_being_edited, edit_history_event=topic_only_edit_history_event, last_edit_time=timestamp, ) changed_messages += messages_list if new_stream is not None: assert stream_being_edited is not None changed_message_ids = [msg.id for msg in changed_messages] if subs_gaining_usermessages: ums_to_create = [] for message_id in changed_message_ids: for user_profile_id in subs_gaining_usermessages: # The fact that the user didn't have a UserMessage originally means we can infer that the user # was not mentioned in the original message (even if mention syntax was present, it would not # take effect for a user who was not subscribed). If we were editing the message's content, we # would rerender the message and then use the new stream's data to determine whether this is # a mention of a subscriber; but as we are not doing so, we choose to preserve the "was this # mention syntax an actual mention" decision made during the original rendering for implementation # simplicity. As a result, the only flag to consider applying here is read. um = UserMessageLite( user_profile_id=user_profile_id, message_id=message_id, flags=UserMessage.flags.read, ) ums_to_create.append(um) bulk_insert_ums(ums_to_create) # Delete UserMessage objects for users who will no # longer have access to these messages. Note: This could be # very expensive, since it's N guest users x M messages. UserMessage.objects.filter( user_profile_id__in=[ sub.user_profile_id for sub in subs_losing_usermessages ], message_id__in=changed_message_ids, ).delete() delete_event: DeleteMessagesEvent = { "type": "delete_message", "message_ids": changed_message_ids, "message_type": "stream", "stream_id": stream_being_edited.id, "topic": orig_topic_name, } delete_event_notify_user_ids = [ sub.user_profile_id for sub in subs_losing_access ] send_event(user_profile.realm, delete_event, delete_event_notify_user_ids) # Reset the Attachment.is_*_public caches for all messages # moved to another stream with different access permissions. if new_stream.invite_only != stream_being_edited.invite_only: Attachment.objects.filter( messages__in=changed_message_ids).update( is_realm_public=None, ) ArchivedAttachment.objects.filter( messages__in=changed_message_ids).update( is_realm_public=None, ) if new_stream.is_web_public != stream_being_edited.is_web_public: Attachment.objects.filter( messages__in=changed_message_ids).update( is_web_public=None, ) ArchivedAttachment.objects.filter( messages__in=changed_message_ids).update( is_web_public=None, ) # This does message.save(update_fields=[...]) save_message_for_edit_use_case(message=target_message) realm_id: Optional[int] = None if stream_being_edited is not None: realm_id = stream_being_edited.realm_id event["message_ids"] = update_to_dict_cache(changed_messages, realm_id) def user_info(um: UserMessage) -> Dict[str, Any]: return { "id": um.user_profile_id, "flags": um.flags_list(), } # The following blocks arranges that users who are subscribed to a # stream and can see history from before they subscribed get # live-update when old messages are edited (e.g. if the user does # a topic edit themself). # # We still don't send an update event to users who are not # subscribed to this stream and don't have a UserMessage row. This # means if a non-subscriber is viewing the narrow, they won't get # a real-time updates. This is a balance between sending # message-edit notifications for every public stream to every user # in the organization (too expansive, and also not what we do for # newly sent messages anyway) and having magical live-updates # where possible. users_to_be_notified = list(map(user_info, ums)) if stream_being_edited is not None: if stream_being_edited.is_history_public_to_subscribers(): subscriptions = get_active_subscriptions_for_stream_id( stream_id, include_deactivated_users=False) # We exclude long-term idle users, since they by # definition have no active clients. subscriptions = subscriptions.exclude( user_profile__long_term_idle=True) # Remove duplicates by excluding the id of users already # in users_to_be_notified list. This is the case where a # user both has a UserMessage row and is a current # Subscriber subscriptions = subscriptions.exclude( user_profile_id__in=[um.user_profile_id for um in ums]) if new_stream is not None: assert delete_event_notify_user_ids is not None subscriptions = subscriptions.exclude( user_profile_id__in=delete_event_notify_user_ids) # All users that are subscribed to the stream must be # notified when a message is edited subscriber_ids = set( subscriptions.values_list("user_profile_id", flat=True)) if new_stream is not None: # TODO: Guest users don't see the new moved topic # unless breadcrumb message for new stream is # enabled. Excluding these users from receiving this # event helps us avoid a error traceback for our # clients. We should figure out a way to inform the # guest users of this new topic if sending a 'message' # event for these messages is not an option. # # Don't send this event to guest subs who are not # subscribers of the old stream but are subscribed to # the new stream; clients will be confused. old_stream_unsubbed_guests = [ sub for sub in subs_to_new_stream if sub.user_profile.is_guest and sub.user_profile_id not in subscriber_ids ] subscriptions = subscriptions.exclude(user_profile_id__in=[ sub.user_profile_id for sub in old_stream_unsubbed_guests ]) subscriber_ids = set( subscriptions.values_list("user_profile_id", flat=True)) users_to_be_notified += list( map(subscriber_info, sorted(list(subscriber_ids)))) # UserTopic updates and the content of notifications depend on # whether we've moved the entire topic, or just part of it. We # make that determination here. moved_all_visible_messages = False if topic_name is not None or new_stream is not None: assert stream_being_edited is not None if propagate_mode == "change_all": moved_all_visible_messages = True else: # With other propagate modes, if the user in fact moved # all messages in the stream, we want to explain it was a # full-topic move. # # For security model reasons, we don't want to allow a # user to take any action that would leak information # about older messages they cannot access (E.g. the only # remaining messages are in a stream without shared # history). The bulk_access_messages call below addresses # that concern. # # bulk_access_messages is inefficient for this task, since # we just want to do the exists() version of this # query. But it's nice to reuse code, and this bulk # operation is likely cheaper than a `GET /messages` # unless the topic has thousands of messages of history. assert stream_being_edited.recipient_id is not None unmoved_messages = messages_for_topic( stream_being_edited.recipient_id, orig_topic_name, ) visible_unmoved_messages = bulk_access_messages( user_profile, unmoved_messages, stream=stream_being_edited) moved_all_visible_messages = len(visible_unmoved_messages) == 0 # Migrate muted topic configuration in the following circumstances: # # * If propagate_mode is change_all, do so unconditionally. # # * If propagate_mode is change_later or change_one, do so when # the acting user has moved the entire topic (as visible to them). # # This rule corresponds to checking moved_all_visible_messages. # # We may want more complex behavior in cases where one appears to # be merging topics (E.g. there are existing messages in the # target topic). if moved_all_visible_messages: assert stream_being_edited is not None assert topic_name is not None or new_stream is not None for muting_user in get_users_muting_topic(stream_being_edited.id, orig_topic_name): # TODO: Ideally, this would be a bulk update operation, # because we are doing database operations in a loop here. # # This loop is only acceptable in production because it is # rare for more than a few users to have muted an # individual topic that is being moved; as of this # writing, no individual topic in Zulip Cloud had been # muted by more than 100 users. if new_stream is not None and muting_user.id in delete_event_notify_user_ids: # If the messages are being moved to a stream the user # cannot access, then we treat this as the # messages/topic being deleted for this user. This is # important for security reasons; we don't want to # give users a UserTopic row in a stream they cannot # access. Unmute the topic for such users. do_unmute_topic(muting_user, stream_being_edited, orig_topic_name) else: # Otherwise, we move the muted topic record for the user. # We call remove_topic_mute rather than do_unmute_topic to # avoid sending two events with new muted topics in # immediate succession; this is correct only because # muted_topics events always send the full set of topics. remove_topic_mute(muting_user, stream_being_edited.id, orig_topic_name) do_mute_topic( muting_user, new_stream if new_stream is not None else stream_being_edited, topic_name if topic_name is not None else orig_topic_name, ignore_duplicate=True, ) send_event(user_profile.realm, event, users_to_be_notified) if len( changed_messages ) > 0 and new_stream is not None and stream_being_edited is not None: # Notify users that the topic was moved. changed_messages_count = len(changed_messages) old_thread_notification_string = None if send_notification_to_old_thread: if moved_all_visible_messages: old_thread_notification_string = gettext_lazy( "This topic was moved to {new_location} by {user}.") elif changed_messages_count == 1: old_thread_notification_string = gettext_lazy( "A message was moved from this topic to {new_location} by {user}." ) else: old_thread_notification_string = gettext_lazy( "{changed_messages_count} messages were moved from this topic to {new_location} by {user}." ) new_thread_notification_string = None if send_notification_to_new_thread: if moved_all_visible_messages: new_thread_notification_string = gettext_lazy( "This topic was moved here from {old_location} by {user}.") elif changed_messages_count == 1: new_thread_notification_string = gettext_lazy( "A message was moved here from {old_location} by {user}.") else: new_thread_notification_string = gettext_lazy( "{changed_messages_count} messages were moved here from {old_location} by {user}." ) send_message_moved_breadcrumbs( user_profile, stream_being_edited, orig_topic_name, old_thread_notification_string, new_stream, topic_name, new_thread_notification_string, changed_messages_count, ) if (topic_name is not None and new_stream is None and content is None and len(changed_messages) > 0): assert stream_being_edited is not None maybe_send_resolve_topic_notifications( user_profile=user_profile, stream=stream_being_edited, old_topic=orig_topic_name, new_topic=topic_name, changed_messages=changed_messages, ) return len(changed_messages)
def build_message_dict( message: Optional[Message], message_id: int, last_edit_time: Optional[datetime.datetime], edit_history: Optional[str], content: str, topic_name: str, pub_date: datetime.datetime, rendered_content: Optional[str], rendered_content_version: Optional[int], sender_id: int, sender_realm_id: int, sending_client_name: str, recipient_id: int, recipient_type: int, recipient_type_id: int, reactions: List[Dict[str, Any]], submessages: List[Dict[str, Any]] ) -> Dict[str, Any]: obj = dict( id = message_id, sender_id = sender_id, content = content, recipient_type_id = recipient_type_id, recipient_type = recipient_type, recipient_id = recipient_id, timestamp = datetime_to_timestamp(pub_date), client = sending_client_name) obj[TOPIC_NAME] = topic_name obj['sender_realm_id'] = sender_realm_id obj['raw_display_recipient'] = get_display_recipient_by_id( recipient_id, recipient_type, recipient_type_id ) obj[TOPIC_LINKS] = bugdown.topic_links(sender_realm_id, topic_name) if last_edit_time is not None: obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time) assert edit_history is not None obj['edit_history'] = ujson.loads(edit_history) if Message.need_to_render_content(rendered_content, rendered_content_version, bugdown.version): if message is None: # We really shouldn't be rendering objects in this method, but there is # a scenario where we upgrade the version of bugdown and fail to run # management commands to re-render historical messages, and then we # need to have side effects. This method is optimized to not need full # blown ORM objects, but the bugdown renderer is unfortunately highly # coupled to Message, and we also need to persist the new rendered content. # If we don't have a message object passed in, we get one here. The cost # of going to the DB here should be overshadowed by the cost of rendering # and updating the row. # TODO: see #1379 to eliminate bugdown dependencies message = Message.objects.select_related().get(id=message_id) assert message is not None # Hint for mypy. # It's unfortunate that we need to have side effects on the message # in some cases. rendered_content = save_message_rendered_content(message, content) if rendered_content is not None: obj['rendered_content'] = rendered_content else: obj['rendered_content'] = ('<p>[Zulip note: Sorry, we could not ' + 'understand the formatting of your message]</p>') if rendered_content is not None: obj['is_me_message'] = Message.is_status_message(content, rendered_content) else: obj['is_me_message'] = False obj['reactions'] = [ReactionDict.build_dict_from_raw_db_row(reaction) for reaction in reactions] obj['submessages'] = submessages return obj
def build_message_dict( apply_markdown, message, message_id, last_edit_time, edit_history, content, subject, pub_date, rendered_content, rendered_content_version, sender_id, sender_email, sender_realm_id, sender_realm_str, sender_avatar_source, sender_avatar_version, sender_is_mirror_dummy, sending_client_name, recipient_id, recipient_type, recipient_type_id, reactions ): # type: (bool, Optional[Message], int, Optional[datetime.datetime], Optional[Text], Text, Text, datetime.datetime, Optional[Text], Optional[int], int, Text, int, Text, Text, int, bool, Text, int, int, int, List[Dict[str, Any]]) -> Dict[str, Any] avatar_url = avatar_url_from_dict(dict( avatar_source=sender_avatar_source, avatar_version=sender_avatar_version, email=sender_email, id=sender_id, realm_id=sender_realm_id)) obj = dict( id = message_id, sender_email = sender_email, sender_realm_str = sender_realm_str, sender_id = sender_id, recipient_type_id = recipient_type_id, recipient_type = recipient_type, recipient_id = recipient_id, subject = subject, timestamp = datetime_to_timestamp(pub_date), avatar_url = avatar_url, client = sending_client_name) obj['raw_display_recipient'] = get_display_recipient_by_id( recipient_id, recipient_type, recipient_type_id ) obj['sender_is_mirror_dummy'] = sender_is_mirror_dummy obj['subject_links'] = bugdown.subject_links(sender_realm_id, subject) if last_edit_time is not None: obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time) assert edit_history is not None obj['edit_history'] = ujson.loads(edit_history) if apply_markdown: if Message.need_to_render_content(rendered_content, rendered_content_version, bugdown.version): if message is None: # We really shouldn't be rendering objects in this method, but there is # a scenario where we upgrade the version of bugdown and fail to run # management commands to re-render historical messages, and then we # need to have side effects. This method is optimized to not need full # blown ORM objects, but the bugdown renderer is unfortunately highly # coupled to Message, and we also need to persist the new rendered content. # If we don't have a message object passed in, we get one here. The cost # of going to the DB here should be overshadowed by the cost of rendering # and updating the row. # TODO: see #1379 to eliminate bugdown dependencies message = Message.objects.select_related().get(id=message_id) assert message is not None # Hint for mypy. # It's unfortunate that we need to have side effects on the message # in some cases. rendered_content = render_markdown(message, content, realm=message.get_realm()) message.rendered_content = rendered_content message.rendered_content_version = bugdown.version message.save_rendered_content() if rendered_content is not None: obj['content'] = rendered_content else: obj['content'] = u'<p>[Zulip note: Sorry, we could not understand the formatting of your message]</p>' obj['content_type'] = 'text/html' else: obj['content'] = content obj['content_type'] = 'text/x-markdown' if rendered_content is not None: obj['is_me_message'] = Message.is_status_message(content, rendered_content) else: obj['is_me_message'] = False obj['reactions'] = [ReactionDict.build_dict_from_raw_db_row(reaction) for reaction in reactions] return obj