def message_header(user_profile: UserProfile, message: Message) -> Dict[str, Any]:
if message.recipient.type == Recipient.PERSONAL:
header = "You and %s" % (message.sender.full_name,)
html_link = personal_narrow_url(
realm=user_profile.realm,
sender=message.sender,
)
header_html = "<a style='color: #ffffff;' href='%s'>%s</a>" % (html_link, header)
elif message.recipient.type == Recipient.HUDDLE:
disp_recipient = get_display_recipient(message.recipient)
assert not isinstance(disp_recipient, str)
other_recipients = [r['full_name'] for r in disp_recipient
if r['id'] != user_profile.id]
header = "You and %s" % (", ".join(other_recipients),)
other_user_ids = [r['id'] for r in disp_recipient
if r['id'] != user_profile.id]
html_link = huddle_narrow_url(
realm=user_profile.realm,
other_user_ids=other_user_ids,
)
header_html = "<a style='color: #ffffff;' href='%s'>%s</a>" % (html_link, header)
else:
stream = Stream.objects.only('id', 'name').get(id=message.recipient.type_id)
header = "%s > %s" % (stream.name, message.topic_name())
stream_link = stream_narrow_url(user_profile.realm, stream)
topic_link = topic_narrow_url(user_profile.realm, stream, message.topic_name())
header_html = "<a href='%s'>%s</a> > <a href='%s'>%s</a>" % (
stream_link, stream.name, topic_link, message.topic_name())
return {"plain": header,
"html": header_html,
"stream_message": message.recipient.type_name() == "stream"}
def test_mention_invalid(self):
sender_user_profile = get_user_profile_by_email("*****@*****.**")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "Hey @**Nonexistent User**"
self.assertEqual(msg.render_markdown(content),
'<p>Hey @<strong>Nonexistent User</strong></p>')
self.assertEqual(msg.mentions_user_ids, set())
def test_mention_everyone(self):
user_profile = get_user_profile_by_email("*****@*****.**")
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "@everyone test"
self.assertEqual(msg.render_markdown(content),
'<p><span class="user-mention" data-user-email="*">@everyone</span> test</p>')
self.assertTrue(msg.mentions_wildcard)
def test_mention_shortname(self):
sender_user_profile = get_user_profile_by_email("*****@*****.**")
user_profile = get_user_profile_by_email("*****@*****.**")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "@**hamlet**"
self.assertEqual(msg.render_markdown(content),
'<p><span class="user-mention" data-user-email="*****@*****.**">@King Hamlet</span></p>')
self.assertEqual(msg.mentions_user_ids, set([user_profile.id]))
def render_message_backend(request, user_profile, content=REQ()):
# type: (HttpRequest, UserProfile, Text) -> HttpResponse
message = Message()
message.sender = user_profile
message.content = content
message.sending_client = request.client
rendered_content = render_markdown(message, content, realm=user_profile.realm)
return json_success({"rendered": rendered_content})
def get_apns_alert_title(message: Message) -> str:
"""
On an iOS notification, this is the first bolded line.
"""
if message.recipient.type == Recipient.HUDDLE:
recipients = cast(List[Dict[str, Any]], get_display_recipient(message.recipient))
return ', '.join(sorted(r['full_name'] for r in recipients))
elif message.is_stream_message():
return "#%s > %s" % (get_display_recipient(message.recipient), message.topic_name(),)
# For personal PMs, we just show the sender name.
return message.sender.full_name
def test_alert_words(self):
user_profile = get_user_profile_by_email("*****@*****.**")
do_set_alert_words(user_profile, ["ALERTWORD", "scaryword"])
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "We have an ALERTWORD day today!"
self.assertEqual(msg.render_markdown(content), "<p>We have an ALERTWORD day today!</p>")
self.assertEqual(msg.user_ids_with_alert_words, set([user_profile.id]))
msg = Message(sender=user_profile, sending_client=get_client("test"))
content = "We have a NOTHINGWORD day today!"
self.assertEqual(msg.render_markdown(content), "<p>We have a NOTHINGWORD day today!</p>")
self.assertEqual(msg.user_ids_with_alert_words, set())
def test_mention_multiple(self):
sender_user_profile = get_user_profile_by_email("*****@*****.**")
hamlet = get_user_profile_by_email("*****@*****.**")
cordelia = get_user_profile_by_email("*****@*****.**")
msg = Message(sender=sender_user_profile, sending_client=get_client("test"))
content = "@**King Hamlet** and @**cordelia**, check this out"
self.assertEqual(msg.render_markdown(content),
'<p>'
'<span class="user-mention" '
'data-user-email="*****@*****.**">@King Hamlet</span> and '
'<span class="user-mention" '
'data-user-email="*****@*****.**">@Cordelia Lear</span>, '
'check this out</p>')
self.assertEqual(msg.mentions_user_ids, set([hamlet.id, cordelia.id]))
def get_alert_from_message(message: Message) -> str:
"""
Determine what alert string to display based on the missed messages.
"""
sender_str = message.sender.full_name
if message.recipient.type == Recipient.HUDDLE and message.trigger == 'private_message':
return "New private group message from %s" % (sender_str,)
elif message.recipient.type == Recipient.PERSONAL and message.trigger == 'private_message':
return "New private message from %s" % (sender_str,)
elif message.is_stream_message() and message.trigger == 'mentioned':
return "New mention from %s" % (sender_str,)
elif (message.is_stream_message() and
(message.trigger == 'stream_push_notify' and message.stream_name)):
return "New stream message from %s in %s" % (sender_str, message.stream_name,)
else:
return "New Zulip mentions and private messages from %s" % (sender_str,)
def archive(request: HttpRequest,
stream_id: int,
topic_name: str) -> HttpResponse:
def get_response(rendered_message_list: List[str],
is_web_public: bool,
stream_name: str) -> HttpResponse:
return render(
request,
'zerver/archive/index.html',
context={
'is_web_public': is_web_public,
'message_list': rendered_message_list,
'stream': stream_name,
'topic': topic_name,
}
)
try:
stream = get_stream_by_id(stream_id)
except JsonableError:
return get_response([], False, '')
if not stream.is_web_public:
return get_response([], False, '')
all_messages = list(Message.objects.select_related(
'sender').filter(recipient__type_id=stream_id, subject=topic_name).order_by('pub_date'))
if not all_messages:
return get_response([], True, stream.name)
rendered_message_list = []
prev_sender = None
for msg in all_messages:
include_sender = False
status_message = Message.is_status_message(msg.content, msg.rendered_content)
if not prev_sender or prev_sender != msg.sender or status_message:
if status_message:
prev_sender = None
else:
prev_sender = msg.sender
include_sender = True
if status_message:
status_message = msg.rendered_content[4+3: -4]
context = {
'sender_full_name': msg.sender.full_name,
'timestampstr': datetime_to_timestamp(msg.last_edit_time
if msg.last_edit_time
else msg.pub_date),
'message_content': msg.rendered_content,
'avatar_url': get_gravatar_url(msg.sender.email, 1),
'include_sender': include_sender,
'status_message': status_message,
}
rendered_msg = loader.render_to_string('zerver/archive/single_message.html', context)
rendered_message_list.append(rendered_msg)
return get_response(rendered_message_list, True, stream.name)
def render_markdown(message, content, realm=None, realm_alert_words=None, message_users=None):
# type: (Message, Text, Optional[Realm], Optional[RealmAlertWords], Set[UserProfile]) -> Text
"""Return HTML for given markdown. Bugdown may add properties to the
message object such as `mentions_user_ids` and `mentions_wildcard`.
These are only on this Django object and are not saved in the
database.
"""
if message_users is None:
message_user_ids = set() # type: Set[int]
else:
message_user_ids = {u.id for u in message_users}
if message is not None:
message.mentions_wildcard = False
message.is_me_message = False
message.mentions_user_ids = set()
message.alert_words = set()
message.links_for_preview = set()
message.outgoing_webhook_bot_triggers = []
if realm is None:
realm = message.get_realm()
possible_words = set() # type: Set[Text]
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
possible_words.update(set(words))
if message is None:
# If we don't have a message, then we are in the compose preview
# codepath, so we know we are dealing with a human.
sent_by_bot = False
else:
sent_by_bot = get_user_profile_by_id(message.sender_id).is_bot
# DO MAIN WORK HERE -- call bugdown to convert
rendered_content = bugdown.convert(content, message=message, message_realm=realm,
possible_words=possible_words,
sent_by_bot=sent_by_bot)
if message is not None:
message.user_ids_with_alert_words = set()
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
if set(words).intersection(message.alert_words):
message.user_ids_with_alert_words.add(user_id)
message.is_me_message = Message.is_status_message(content, rendered_content)
return rendered_content
def render_markdown(message: Message,
content: str,
realm: Optional[Realm]=None,
realm_alert_words: Optional[RealmAlertWords]=None,
user_ids: Optional[Set[int]]=None,
mention_data: Optional[bugdown.MentionData]=None,
email_gateway: Optional[bool]=False) -> str:
"""Return HTML for given markdown. Bugdown may add properties to the
message object such as `mentions_user_ids`, `mentions_user_group_ids`, and
`mentions_wildcard`. These are only on this Django object and are not
saved in the database.
"""
if user_ids is None:
message_user_ids = set() # type: Set[int]
else:
message_user_ids = user_ids
message.mentions_wildcard = False
message.mentions_user_ids = set()
message.mentions_user_group_ids = set()
message.alert_words = set()
message.links_for_preview = set()
if realm is None:
realm = message.get_realm()
possible_words = set() # type: Set[str]
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
possible_words.update(set(words))
sent_by_bot = get_user_profile_by_id(message.sender_id).is_bot
# DO MAIN WORK HERE -- call bugdown to convert
rendered_content = bugdown.convert(
content,
message=message,
message_realm=realm,
possible_words=possible_words,
sent_by_bot=sent_by_bot,
mention_data=mention_data,
email_gateway=email_gateway
)
if message is not None:
message.user_ids_with_alert_words = set()
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
if set(words).intersection(message.alert_words):
message.user_ids_with_alert_words.add(user_id)
return rendered_content
def build_message(topic_name: str, pub_date: float, message_id: int, content: str,
rendered_content: Optional[str], user_id: int, recipient_id: int,
has_image: bool=False, has_link: bool=False,
has_attachment: bool=True) -> ZerverFieldsT:
zulip_message = Message(
rendered_content_version=1, # this is Zulip specific
pub_date=pub_date,
id=message_id,
content=content,
rendered_content=rendered_content,
has_image=has_image,
has_attachment=has_attachment,
has_link=has_link)
zulip_message.set_topic_name(topic_name)
zulip_message_dict = model_to_dict(zulip_message,
exclude=['recipient', 'sender', 'sending_client'])
zulip_message_dict['sender'] = user_id
zulip_message_dict['sending_client'] = 1
zulip_message_dict['recipient'] = recipient_id
return zulip_message_dict
def do_render_markdown(message: Message,
content: str,
realm: Realm,
message_user_ids: Set[int],
sent_by_bot: bool,
translate_emoticons: bool,
realm_alert_words_automaton: Optional[ahocorasick.Automaton]=None,
mention_data: Optional[bugdown.MentionData]=None,
email_gateway: Optional[bool]=False) -> str:
"""Return HTML for given markdown. Bugdown may add properties to the
message object such as `mentions_user_ids`, `mentions_user_group_ids`, and
`mentions_wildcard`. These are only on this Django object and are not
saved in the database.
"""
message.mentions_wildcard = False
message.mentions_user_ids = set()
message.mentions_user_group_ids = set()
message.alert_words = set()
message.links_for_preview = set()
message.user_ids_with_alert_words = set()
# DO MAIN WORK HERE -- call bugdown to convert
rendered_content = bugdown.convert(
content,
realm_alert_words_automaton=realm_alert_words_automaton,
message=message,
message_realm=realm,
sent_by_bot=sent_by_bot,
translate_emoticons=translate_emoticons,
mention_data=mention_data,
email_gateway=email_gateway
)
return rendered_content
def re_render_content_for_management_command(message):
# type: (Message) -> None
'''
Please avoid using this function, as its only used in a management command that
is somewhat deprecated.
'''
assert Message.need_to_render_content(message.rendered_content,
message.rendered_content_version,
bugdown.version)
rendered_content = render_markdown(message, message.content)
message.rendered_content = rendered_content
message.rendered_content_version = bugdown.version
message.save_rendered_content()
def render_markdown(message, content, realm_id=None, realm_alert_words=None, message_users=None):
# type: (Message, Text, Optional[int], Optional[RealmAlertWords], Set[UserProfile]) -> Text
"""Return HTML for given markdown. Bugdown may add properties to the
message object such as `mentions_user_ids` and `mentions_wildcard`.
These are only on this Django object and are not saved in the
database.
"""
if message_users is None:
message_user_ids = set() # type: Set[int]
else:
message_user_ids = {u.id for u in message_users}
if message is not None:
message.mentions_wildcard = False
message.is_me_message = False
message.mentions_user_ids = set()
message.alert_words = set()
message.links_for_preview = set()
if realm_id is None:
realm_id = message.sender.realm_id
if message.sending_client.name == "zephyr_mirror" and message.sender.realm.is_zephyr_mirror_realm:
# Use slightly customized Markdown processor for content
# delivered via zephyr_mirror
realm_id = bugdown.ZEPHYR_MIRROR_BUGDOWN_KEY
possible_words = set() # type: Set[Text]
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
possible_words.update(set(words))
# DO MAIN WORK HERE -- call bugdown to convert
rendered_content = bugdown.convert(content, realm_filters_key=realm_id, message=message,
possible_words=possible_words)
if message is not None:
message.user_ids_with_alert_words = set()
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
if set(words).intersection(message.alert_words):
message.user_ids_with_alert_words.add(user_id)
message.is_me_message = Message.is_status_message(content, rendered_content)
return rendered_content
def construct_message(self, scheduled_message: ScheduledMessage) -> Dict[str, Any]:
message = Message()
original_sender = scheduled_message.sender
message.content = scheduled_message.content
message.recipient = scheduled_message.recipient
message.subject = scheduled_message.subject
message.pub_date = timezone_now()
message.sending_client = scheduled_message.sending_client
delivery_type = scheduled_message.delivery_type
if delivery_type == ScheduledMessage.SEND_LATER:
message.sender = original_sender
elif delivery_type == ScheduledMessage.REMIND:
message.sender = get_user(settings.REMINDER_BOT, original_sender.realm)
return {'message': message, 'stream': scheduled_message.stream,
'realm': scheduled_message.realm}
def get_common_payload(message: Message) -> Dict[str, Any]:
data = {} # type: Dict[str, Any]
# These will let the app support logging into multiple realms and servers.
data['server'] = settings.EXTERNAL_HOST
data['realm_id'] = message.sender.realm.id
# `sender_id` is preferred, but some existing versions use `sender_email`.
data['sender_id'] = message.sender.id
data['sender_email'] = message.sender.email
if message.is_stream_message():
data['recipient_type'] = "stream"
data['stream'] = get_display_recipient(message.recipient)
data['topic'] = message.subject
else:
data['recipient_type'] = "private"
return data
def get_message_payload(message: Message) -> Dict[str, Any]:
'''Common fields for `message` payloads, for all platforms.'''
data = get_base_payload(message.sender.realm)
# `sender_id` is preferred, but some existing versions use `sender_email`.
data['sender_id'] = message.sender.id
data['sender_email'] = message.sender.email
if message.recipient.type == Recipient.STREAM:
data['recipient_type'] = "stream"
data['stream'] = get_display_recipient(message.recipient)
data['topic'] = message.topic_name()
elif message.recipient.type == Recipient.HUDDLE:
data['recipient_type'] = "private"
data['pm_users'] = huddle_users(message.recipient.id)
else: # Recipient.PERSONAL
data['recipient_type'] = "private"
return data
def to_dict_uncached_helper(message: Message) -> Dict[str, Any]:
return MessageDict.build_message_dict(
message = message,
message_id = message.id,
last_edit_time = message.last_edit_time,
edit_history = message.edit_history,
content = message.content,
topic_name = message.topic_name(),
pub_date = message.pub_date,
rendered_content = message.rendered_content,
rendered_content_version = message.rendered_content_version,
sender_id = message.sender.id,
sender_realm_id = message.sender.realm_id,
sending_client_name = message.sending_client.name,
recipient_id = message.recipient.id,
recipient_type = message.recipient.type,
recipient_type_id = message.recipient.type_id,
reactions = Reaction.get_raw_db_rows([message.id]),
submessages = SubMessage.get_raw_db_rows([message.id]),
)
def render_markdown(message: Message,
content: str,
realm: Optional[Realm]=None,
realm_alert_words: Optional[RealmAlertWords]=None,
user_ids: Optional[Set[int]]=None,
mention_data: Optional[bugdown.MentionData]=None,
email_gateway: Optional[bool]=False) -> str:
'''
This is basically just a wrapper for do_render_markdown.
'''
if user_ids is None:
message_user_ids = set() # type: Set[int]
else:
message_user_ids = user_ids
if realm is None:
realm = message.get_realm()
if realm_alert_words is None:
realm_alert_words = dict()
sender = get_user_profile_by_id(message.sender_id)
sent_by_bot = sender.is_bot
translate_emoticons = sender.translate_emoticons
rendered_content = do_render_markdown(
message=message,
content=content,
realm=realm,
realm_alert_words=realm_alert_words,
message_user_ids=message_user_ids,
sent_by_bot=sent_by_bot,
translate_emoticons=translate_emoticons,
mention_data=mention_data,
email_gateway=email_gateway,
)
return rendered_content
def do_render_markdown(message: Message,
content: str,
realm: Realm,
realm_alert_words: RealmAlertWords,
message_user_ids: Set[int],
sent_by_bot: bool,
translate_emoticons: bool,
mention_data: Optional[bugdown.MentionData]=None,
email_gateway: Optional[bool]=False) -> str:
"""Return HTML for given markdown. Bugdown may add properties to the
message object such as `mentions_user_ids`, `mentions_user_group_ids`, and
`mentions_wildcard`. These are only on this Django object and are not
saved in the database.
"""
message.mentions_wildcard = False
message.mentions_user_ids = set()
message.mentions_user_group_ids = set()
message.alert_words = set()
message.links_for_preview = set()
possible_words = set() # type: Set[str]
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
possible_words.update(set(words))
# DO MAIN WORK HERE -- call bugdown to convert
rendered_content = bugdown.convert(
content,
message=message,
message_realm=realm,
possible_words=possible_words,
sent_by_bot=sent_by_bot,
translate_emoticons=translate_emoticons,
mention_data=mention_data,
email_gateway=email_gateway
)
message.user_ids_with_alert_words = set()
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
if set(words).intersection(message.alert_words):
message.user_ids_with_alert_words.add(user_id)
return rendered_content
def get_common_payload(message: Message) -> Dict[str, Any]:
data = {} # type: Dict[str, Any]
# These will let the app support logging into multiple realms and servers.
data['server'] = settings.EXTERNAL_HOST
data['realm_id'] = message.sender.realm.id
data['realm_uri'] = message.sender.realm.uri
# `sender_id` is preferred, but some existing versions use `sender_email`.
data['sender_id'] = message.sender.id
data['sender_email'] = message.sender.email
if message.recipient.type == Recipient.STREAM:
data['recipient_type'] = "stream"
data['stream'] = get_display_recipient(message.recipient)
data['topic'] = message.topic_name()
elif message.recipient.type == Recipient.HUDDLE:
data['recipient_type'] = "private"
data['pm_users'] = huddle_users(message.recipient.id)
else: # Recipient.PERSONAL
data['recipient_type'] = "private"
return data
def do_render_markdown(message: Message,
content: str,
realm: Realm,
realm_alert_words: RealmAlertWords,
message_user_ids: Set[int],
sent_by_bot: bool,
translate_emoticons: bool,
mention_data: Optional[bugdown.MentionData] = None,
email_gateway: Optional[bool] = False) -> str:
"""Return HTML for given markdown. Bugdown may add properties to the
message object such as `mentions_user_ids`, `mentions_user_group_ids`, and
`mentions_wildcard`. These are only on this Django object and are not
saved in the database.
"""
message.mentions_wildcard = False
message.mentions_user_ids = set()
message.mentions_user_group_ids = set()
message.alert_words = set()
message.links_for_preview = set()
possible_words = set() # type: Set[str]
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
possible_words.update(set(words))
# DO MAIN WORK HERE -- call bugdown to convert
rendered_content = bugdown.convert(content,
message=message,
message_realm=realm,
possible_words=possible_words,
sent_by_bot=sent_by_bot,
translate_emoticons=translate_emoticons,
mention_data=mention_data,
email_gateway=email_gateway)
message.user_ids_with_alert_words = set()
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
if set(words).intersection(message.alert_words):
message.user_ids_with_alert_words.add(user_id)
return rendered_content
def message_header(user_profile: UserProfile, message: Message) -> Dict[str, Any]:
disp_recipient = get_display_recipient(message.recipient)
if message.recipient.type == Recipient.PERSONAL:
header = u"You and %s" % (message.sender.full_name,)
html_link = pm_narrow_url(user_profile.realm, [message.sender.email])
header_html = u"<a style='color: #ffffff;' href='%s'>%s</a>" % (html_link, header)
elif message.recipient.type == Recipient.HUDDLE:
assert not isinstance(disp_recipient, Text)
other_recipients = [r['full_name'] for r in disp_recipient
if r['email'] != user_profile.email]
header = u"You and %s" % (", ".join(other_recipients),)
html_link = pm_narrow_url(user_profile.realm, [r["email"] for r in disp_recipient
if r["email"] != user_profile.email])
header_html = u"<a style='color: #ffffff;' href='%s'>%s</a>" % (html_link, header)
else:
assert isinstance(disp_recipient, Text)
header = u"%s > %s" % (disp_recipient, message.topic_name())
stream_link = stream_narrow_url(user_profile.realm, disp_recipient)
topic_link = topic_narrow_url(user_profile.realm, disp_recipient, message.subject)
header_html = u"<a href='%s'>%s</a> > <a href='%s'>%s</a>" % (
stream_link, disp_recipient, topic_link, message.subject)
return {"plain": header,
"html": header_html,
"stream_message": message.recipient.type_name() == "stream"}
def get_narrow_url(user_profile: UserProfile,
message: Message,
display_recipient: Optional[DisplayRecipientT] = None,
stream: Optional[Stream] = None) -> str:
"""The display_recipient and stream arguments are optional. If not
provided, we'll compute them from the message; they exist as a
performance optimization for cases where the caller needs those
data too.
"""
if message.recipient.type == Recipient.PERSONAL:
assert stream is None
assert display_recipient is None
return personal_narrow_url(
realm=user_profile.realm,
sender=message.sender,
)
elif message.recipient.type == Recipient.HUDDLE:
assert stream is None
if display_recipient is None:
display_recipient = get_display_recipient(message.recipient)
assert display_recipient is not None
assert not isinstance(display_recipient, str)
other_user_ids = [
r['id'] for r in display_recipient if r['id'] != user_profile.id
]
return huddle_narrow_url(
realm=user_profile.realm,
other_user_ids=other_user_ids,
)
else:
assert display_recipient is None
if stream is None:
stream = Stream.objects.only(
'id', 'name').get(id=message.recipient.type_id)
return topic_narrow_url(user_profile.realm, stream,
message.topic_name())
def do_claim_attachments(message: Message,
potential_path_ids: List[str]) -> bool:
claimed = False
for path_id in potential_path_ids:
user_profile = message.sender
is_message_realm_public = False
is_message_web_public = False
if message.is_stream_message():
stream = Stream.objects.get(id=message.recipient.type_id)
is_message_realm_public = stream.is_public()
is_message_web_public = stream.is_web_public
if not validate_attachment_request(user_profile, path_id):
# Technically, there are 2 cases here:
# * The user put something in their message that has the form
# of an upload, but doesn't correspond to a file that doesn't
# exist. validate_attachment_request will return None.
# * The user is trying to send a link to a file they don't have permission to
# access themselves. validate_attachment_request will return False.
#
# Either case is unusual and suggests a UI bug that got
# the user in this situation, so we log in these cases.
logging.warning(
"User %s tried to share upload %s in message %s, but lacks permission",
user_profile.id,
path_id,
message.id,
)
continue
claimed = True
attachment = claim_attachment(user_profile, path_id, message,
is_message_realm_public,
is_message_web_public)
notify_attachment_update(user_profile, "update", attachment.to_dict())
return claimed
def render_markdown(message: Message,
content: str,
realm: Optional[Realm]=None,
realm_alert_words_automaton: Optional[ahocorasick.Automaton]=None,
user_ids: Optional[Set[int]]=None,
mention_data: Optional[bugdown.MentionData]=None,
email_gateway: Optional[bool]=False) -> str:
'''
This is basically just a wrapper for do_render_markdown.
'''
if user_ids is None:
message_user_ids = set() # type: Set[int]
else:
message_user_ids = user_ids
if realm is None:
realm = message.get_realm()
sender = get_user_profile_by_id(message.sender_id)
sent_by_bot = sender.is_bot
translate_emoticons = sender.translate_emoticons
rendered_content = do_render_markdown(
message=message,
content=content,
realm=realm,
realm_alert_words_automaton=realm_alert_words_automaton,
message_user_ids=message_user_ids,
sent_by_bot=sent_by_bot,
translate_emoticons=translate_emoticons,
mention_data=mention_data,
email_gateway=email_gateway,
)
return rendered_content
def message_header(user_profile: UserProfile,
message: Message) -> Dict[str, Any]:
if message.recipient.type == Recipient.PERSONAL:
header = "You and %s" % (message.sender.full_name, )
html_link = pm_narrow_url(user_profile.realm,
[message.sender.email])
header_html = "<a style='color: #ffffff;' href='%s'>%s</a>" % (
html_link, header)
elif message.recipient.type == Recipient.HUDDLE:
disp_recipient = get_display_recipient(message.recipient)
assert not isinstance(disp_recipient, str)
other_recipients = [
r['full_name'] for r in disp_recipient
if r['email'] != user_profile.email
]
header = "You and %s" % (", ".join(other_recipients), )
html_link = pm_narrow_url(user_profile.realm, [
r["email"]
for r in disp_recipient if r["email"] != user_profile.email
])
header_html = "<a style='color: #ffffff;' href='%s'>%s</a>" % (
html_link, header)
else:
stream = Stream.objects.only(
'id', 'name').get(id=message.recipient.type_id)
header = "%s > %s" % (stream.name, message.topic_name())
stream_link = stream_narrow_url(user_profile.realm, stream)
topic_link = topic_narrow_url(user_profile.realm, stream,
message.subject)
header_html = "<a href='%s'>%s</a> > <a href='%s'>%s</a>" % (
stream_link, stream.name, topic_link, message.subject)
return {
"plain": header,
"html": header_html,
"stream_message": message.recipient.type_name() == "stream"
}
def test_bulk_message_fetching(self) -> None:
sender = self.example_user("othello")
receiver = self.example_user("hamlet")
pm_recipient = Recipient.objects.get(type_id=receiver.id, type=Recipient.PERSONAL)
stream_name = "Çiğdem"
stream = self.make_stream(stream_name)
stream_recipient = Recipient.objects.get(type_id=stream.id, type=Recipient.STREAM)
sending_client = make_client(name="test suite")
ids = []
for i in range(300):
for recipient in [pm_recipient, stream_recipient]:
message = Message(
sender=sender,
recipient=recipient,
content=f"whatever {i}",
rendered_content="DOES NOT MATTER",
rendered_content_version=markdown_version,
date_sent=timezone_now(),
sending_client=sending_client,
last_edit_time=timezone_now(),
edit_history="[]",
)
message.set_topic_name("whatever")
message.save()
ids.append(message.id)
Reaction.objects.create(
user_profile=sender, message=message, emoji_name="simple_smile"
)
num_ids = len(ids)
self.assertTrue(num_ids >= 600)
flush_per_request_caches()
with queries_captured() as queries:
rows = list(MessageDict.get_raw_db_rows(ids))
objs = [MessageDict.build_dict_from_raw_db_row(row) for row in rows]
MessageDict.post_process_dicts(objs, apply_markdown=False, client_gravatar=False)
self.assert_length(queries, 7)
self.assertEqual(len(rows), num_ids)
def test_sew_messages_and_reaction(self) -> None:
sender = self.example_user("othello")
receiver = self.example_user("hamlet")
pm_recipient = Recipient.objects.get(type_id=receiver.id,
type=Recipient.PERSONAL)
stream_name = "Çiğdem"
stream = self.make_stream(stream_name)
stream_recipient = Recipient.objects.get(type_id=stream.id,
type=Recipient.STREAM)
sending_client = make_client(name="test suite")
needed_ids = []
for i in range(5):
for recipient in [pm_recipient, stream_recipient]:
message = Message(
sender=sender,
recipient=recipient,
content=f"whatever {i}",
date_sent=timezone_now(),
sending_client=sending_client,
last_edit_time=timezone_now(),
edit_history="[]",
)
message.set_topic_name("whatever")
message.save()
needed_ids.append(message.id)
reaction = Reaction(user_profile=sender,
message=message,
emoji_name="simple_smile")
reaction.save()
messages = Message.objects.filter(id__in=needed_ids).values(
*["id", "content"])
reactions = Reaction.get_raw_db_rows(needed_ids)
tied_data = sew_messages_and_reactions(messages, reactions)
for data in tied_data:
self.assert_length(data["reactions"], 1)
self.assertEqual(data["reactions"][0]["emoji_name"],
"simple_smile")
self.assertTrue(data["id"])
self.assertTrue(data["content"])
def test_sew_messages_and_reaction(self) -> None:
sender = self.example_user('othello')
receiver = self.example_user('hamlet')
pm_recipient = Recipient.objects.get(type_id=receiver.id,
type=Recipient.PERSONAL)
stream_name = 'Çiğdem'
stream = self.make_stream(stream_name)
stream_recipient = Recipient.objects.get(type_id=stream.id,
type=Recipient.STREAM)
sending_client = make_client(name="test suite")
needed_ids = []
for i in range(5):
for recipient in [pm_recipient, stream_recipient]:
message = Message(
sender=sender,
recipient=recipient,
content=f'whatever {i}',
date_sent=timezone_now(),
sending_client=sending_client,
last_edit_time=timezone_now(),
edit_history='[]',
)
message.set_topic_name('whatever')
message.save()
needed_ids.append(message.id)
reaction = Reaction(user_profile=sender,
message=message,
emoji_name='simple_smile')
reaction.save()
messages = Message.objects.filter(id__in=needed_ids).values(
*['id', 'content'])
reactions = Reaction.get_raw_db_rows(needed_ids)
tied_data = sew_messages_and_reactions(messages, reactions)
for data in tied_data:
self.assertEqual(len(data['reactions']), 1)
self.assertEqual(data['reactions'][0]['emoji_name'],
'simple_smile')
self.assertTrue(data['id'])
self.assertTrue(data['content'])
def create_test_message(topic: str) -> int:
# TODO: Clean this up to send messages the normal way.
hamlet = self.example_user("hamlet")
message = Message(
sender=hamlet,
recipient=recipient,
content="whatever",
date_sent=timezone_now(),
sending_client=get_client("whatever"),
)
message.set_topic_name(topic)
message.save()
UserMessage.objects.create(
user_profile=user_profile,
message=message,
flags=0,
)
return message.id
def build_message_dict(apply_markdown, message, message_id, last_edit_time,
edit_history, content, subject, pub_date,
rendered_content, rendered_content_version,
sender_id, sender_email, sender_realm_id,
sender_realm_str, sender_full_name,
sender_short_name, sender_avatar_source,
sender_avatar_version, sender_is_mirror_dummy,
sending_client_name, recipient_id, recipient_type,
recipient_type_id, reactions):
# type: (bool, Optional[Message], int, Optional[datetime.datetime], Optional[Text], Text, Text, datetime.datetime, Optional[Text], Optional[int], int, Text, int, Text, Text, Text, Text, int, bool, Text, int, int, int, List[Dict[str, Any]]) -> Dict[str, Any]
avatar_url = get_avatar_url(sender_avatar_source, sender_email,
sender_avatar_version)
display_recipient = get_display_recipient_by_id(
recipient_id, recipient_type, recipient_type_id)
if recipient_type == Recipient.STREAM:
display_type = "stream"
elif recipient_type in (Recipient.HUDDLE, Recipient.PERSONAL):
assert not isinstance(display_recipient, Text)
display_type = "private"
if len(display_recipient) == 1:
# add the sender in if this isn't a message between
# someone and his self, preserving ordering
recip = {
'email': sender_email,
'full_name': sender_full_name,
'short_name': sender_short_name,
'id': sender_id,
'is_mirror_dummy': sender_is_mirror_dummy
}
if recip['email'] < display_recipient[0]['email']:
display_recipient = [recip, display_recipient[0]]
elif recip['email'] > display_recipient[0]['email']:
display_recipient = [display_recipient[0], recip]
obj = dict(
id=message_id,
sender_email=sender_email,
sender_full_name=sender_full_name,
sender_short_name=sender_short_name,
sender_realm_str=sender_realm_str,
sender_id=sender_id,
type=display_type,
display_recipient=display_recipient,
recipient_id=recipient_id,
subject=subject,
timestamp=datetime_to_timestamp(pub_date),
gravatar_hash=gravatar_hash(sender_email), # Deprecated June 2013
avatar_url=avatar_url,
client=sending_client_name)
if obj['type'] == 'stream':
obj['stream_id'] = recipient_type_id
obj['subject_links'] = bugdown.subject_links(sender_realm_id, subject)
if last_edit_time is not None:
obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
obj['edit_history'] = ujson.loads(edit_history)
if apply_markdown:
if Message.need_to_render_content(rendered_content,
rendered_content_version,
bugdown.version):
if message is None:
# We really shouldn't be rendering objects in this method, but there is
# a scenario where we upgrade the version of bugdown and fail to run
# management commands to re-render historical messages, and then we
# need to have side effects. This method is optimized to not need full
# blown ORM objects, but the bugdown renderer is unfortunately highly
# coupled to Message, and we also need to persist the new rendered content.
# If we don't have a message object passed in, we get one here. The cost
# of going to the DB here should be overshadowed by the cost of rendering
# and updating the row.
# TODO: see #1379 to eliminate bugdown dependencies
message = Message.objects.select_related().get(
id=message_id)
# It's unfortunate that we need to have side effects on the message
# in some cases.
rendered_content = render_markdown(message,
content,
realm=message.get_realm())
message.rendered_content = rendered_content
message.rendered_content_version = bugdown.version
message.save_rendered_content()
if rendered_content is not None:
obj['content'] = rendered_content
else:
obj['content'] = u'<p>[Zulip note: Sorry, we could not understand the formatting of your message]</p>'
obj['content_type'] = 'text/html'
else:
obj['content'] = content
obj['content_type'] = 'text/x-markdown'
obj['reactions'] = [
ReactionDict.build_dict_from_raw_db_row(reaction)
for reaction in reactions
]
return obj
def archive(request: HttpRequest, stream_id: int,
topic_name: str) -> HttpResponse:
def get_response(rendered_message_list: List[str], is_web_public: bool,
stream_name: str) -> HttpResponse:
return render(request,
'zerver/archive/index.html',
context={
'is_web_public': is_web_public,
'message_list': rendered_message_list,
'stream': stream_name,
'topic': topic_name,
})
try:
stream = get_stream_by_id(stream_id)
except JsonableError:
return get_response([], False, '')
if not stream.is_web_public:
return get_response([], False, '')
all_messages = list(
Message.objects.select_related('sender').filter(
recipient__type_id=stream_id,
subject=topic_name).order_by('pub_date'))
if not all_messages:
return get_response([], True, stream.name)
rendered_message_list = []
prev_sender = None
for msg in all_messages:
include_sender = False
status_message = Message.is_status_message(msg.content,
msg.rendered_content)
if not prev_sender or prev_sender != msg.sender or status_message:
if status_message:
prev_sender = None
else:
prev_sender = msg.sender
include_sender = True
if status_message:
status_message = msg.rendered_content[4 + 3:-4]
context = {
'sender_full_name':
msg.sender.full_name,
'timestampstr':
datetime_to_timestamp(
msg.last_edit_time if msg.last_edit_time else msg.pub_date),
'message_content':
msg.rendered_content,
'avatar_url':
get_gravatar_url(msg.sender.email, 1),
'include_sender':
include_sender,
'status_message':
status_message,
}
rendered_msg = loader.render_to_string(
'zerver/archive/single_message.html', context)
rendered_message_list.append(rendered_msg)
return get_response(rendered_message_list, True, stream.name)
def render_markdown(message: Message,
content: Text,
realm: Optional[Realm]=None,
realm_alert_words: Optional[RealmAlertWords]=None,
user_ids: Optional[Set[int]]=None,
mention_data: Optional[bugdown.MentionData]=None,
email_gateway: Optional[bool]=False) -> Text:
"""Return HTML for given markdown. Bugdown may add properties to the
message object such as `mentions_user_ids`, `mentions_user_group_ids`, and
`mentions_wildcard`. These are only on this Django object and are not
saved in the database.
"""
if user_ids is None:
message_user_ids = set() # type: Set[int]
else:
message_user_ids = user_ids
if message is not None:
message.mentions_wildcard = False
message.mentions_user_ids = set()
message.mentions_user_group_ids = set()
message.alert_words = set()
message.links_for_preview = set()
if realm is None:
realm = message.get_realm()
possible_words = set() # type: Set[Text]
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
possible_words.update(set(words))
if message is None:
# If we don't have a message, then we are in the compose preview
# codepath, so we know we are dealing with a human.
sent_by_bot = False
else:
sent_by_bot = get_user_profile_by_id(message.sender_id).is_bot
# DO MAIN WORK HERE -- call bugdown to convert
rendered_content = bugdown.convert(
content,
message=message,
message_realm=realm,
possible_words=possible_words,
sent_by_bot=sent_by_bot,
mention_data=mention_data,
email_gateway=email_gateway
)
if message is not None:
message.user_ids_with_alert_words = set()
if realm_alert_words is not None:
for user_id, words in realm_alert_words.items():
if user_id in message_user_ids:
if set(words).intersection(message.alert_words):
message.user_ids_with_alert_words.add(user_id)
return rendered_content
def build_message_dict(
apply_markdown,
message,
message_id,
last_edit_time,
edit_history,
content,
subject,
pub_date,
rendered_content,
rendered_content_version,
sender_id,
sender_email,
sender_realm_id,
sender_realm_str,
sender_avatar_source,
sender_avatar_version,
sender_is_mirror_dummy,
sending_client_name,
recipient_id,
recipient_type,
recipient_type_id,
reactions
):
# type: (bool, Optional[Message], int, Optional[datetime.datetime], Optional[Text], Text, Text, datetime.datetime, Optional[Text], Optional[int], int, Text, int, Text, Text, int, bool, Text, int, int, int, List[Dict[str, Any]]) -> Dict[str, Any]
avatar_url = avatar_url_from_dict(dict(
avatar_source=sender_avatar_source,
avatar_version=sender_avatar_version,
email=sender_email,
id=sender_id,
realm_id=sender_realm_id))
obj = dict(
id = message_id,
sender_email = sender_email,
sender_realm_str = sender_realm_str,
sender_id = sender_id,
recipient_type_id = recipient_type_id,
recipient_type = recipient_type,
recipient_id = recipient_id,
subject = subject,
timestamp = datetime_to_timestamp(pub_date),
avatar_url = avatar_url,
client = sending_client_name)
obj['raw_display_recipient'] = get_display_recipient_by_id(
recipient_id,
recipient_type,
recipient_type_id
)
obj['sender_is_mirror_dummy'] = sender_is_mirror_dummy
obj['subject_links'] = bugdown.subject_links(sender_realm_id, subject)
if last_edit_time is not None:
obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
assert edit_history is not None
obj['edit_history'] = ujson.loads(edit_history)
if apply_markdown:
if Message.need_to_render_content(rendered_content, rendered_content_version, bugdown.version):
if message is None:
# We really shouldn't be rendering objects in this method, but there is
# a scenario where we upgrade the version of bugdown and fail to run
# management commands to re-render historical messages, and then we
# need to have side effects. This method is optimized to not need full
# blown ORM objects, but the bugdown renderer is unfortunately highly
# coupled to Message, and we also need to persist the new rendered content.
# If we don't have a message object passed in, we get one here. The cost
# of going to the DB here should be overshadowed by the cost of rendering
# and updating the row.
# TODO: see #1379 to eliminate bugdown dependencies
message = Message.objects.select_related().get(id=message_id)
assert message is not None # Hint for mypy.
# It's unfortunate that we need to have side effects on the message
# in some cases.
rendered_content = render_markdown(message, content, realm=message.get_realm())
message.rendered_content = rendered_content
message.rendered_content_version = bugdown.version
message.save_rendered_content()
if rendered_content is not None:
obj['content'] = rendered_content
else:
obj['content'] = u'<p>[Zulip note: Sorry, we could not understand the formatting of your message]</p>'
obj['content_type'] = 'text/html'
else:
obj['content'] = content
obj['content_type'] = 'text/x-markdown'
if rendered_content is not None:
obj['is_me_message'] = Message.is_status_message(content, rendered_content)
else:
obj['is_me_message'] = False
obj['reactions'] = [ReactionDict.build_dict_from_raw_db_row(reaction)
for reaction in reactions]
return obj
def save_message_rendered_content(message: Message, content: str) -> str:
rendered_content = render_markdown(message, content, realm=message.get_realm())
message.rendered_content = rendered_content
message.rendered_content_version = bugdown.version
message.save_rendered_content()
return rendered_content
def filter_by_exact_message_topic(query: QuerySet,
message: Message) -> QuerySet:
topic_name = message.topic_name()
return query.filter(subject=topic_name)
def save_message_rendered_content(message: Message, content: str) -> str:
rendered_content = render_markdown(message, content, realm=message.get_realm())
message.rendered_content = rendered_content
message.rendered_content_version = bugdown.version
message.save_rendered_content()
return rendered_content
def build_message_dict(
message: Optional[Message],
message_id: int,
last_edit_time: Optional[datetime.datetime],
edit_history: Optional[str],
content: str,
topic_name: str,
pub_date: datetime.datetime,
rendered_content: Optional[str],
rendered_content_version: Optional[int],
sender_id: int,
sender_realm_id: int,
sending_client_name: str,
recipient_id: int,
recipient_type: int,
recipient_type_id: int,
reactions: List[Dict[str, Any]],
submessages: List[Dict[str, Any]]
) -> Dict[str, Any]:
obj = dict(
id = message_id,
sender_id = sender_id,
content = content,
recipient_type_id = recipient_type_id,
recipient_type = recipient_type,
recipient_id = recipient_id,
timestamp = datetime_to_timestamp(pub_date),
client = sending_client_name)
obj[TOPIC_NAME] = topic_name
obj['sender_realm_id'] = sender_realm_id
obj['raw_display_recipient'] = get_display_recipient_by_id(
recipient_id,
recipient_type,
recipient_type_id
)
obj[TOPIC_LINKS] = bugdown.topic_links(sender_realm_id, topic_name)
if last_edit_time is not None:
obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
assert edit_history is not None
obj['edit_history'] = ujson.loads(edit_history)
if Message.need_to_render_content(rendered_content, rendered_content_version, bugdown.version):
if message is None:
# We really shouldn't be rendering objects in this method, but there is
# a scenario where we upgrade the version of bugdown and fail to run
# management commands to re-render historical messages, and then we
# need to have side effects. This method is optimized to not need full
# blown ORM objects, but the bugdown renderer is unfortunately highly
# coupled to Message, and we also need to persist the new rendered content.
# If we don't have a message object passed in, we get one here. The cost
# of going to the DB here should be overshadowed by the cost of rendering
# and updating the row.
# TODO: see #1379 to eliminate bugdown dependencies
message = Message.objects.select_related().get(id=message_id)
assert message is not None # Hint for mypy.
# It's unfortunate that we need to have side effects on the message
# in some cases.
rendered_content = save_message_rendered_content(message, content)
if rendered_content is not None:
obj['rendered_content'] = rendered_content
else:
obj['rendered_content'] = ('<p>[Zulip note: Sorry, we could not ' +
'understand the formatting of your message]</p>')
if rendered_content is not None:
obj['is_me_message'] = Message.is_status_message(content, rendered_content)
else:
obj['is_me_message'] = False
obj['reactions'] = [ReactionDict.build_dict_from_raw_db_row(reaction)
for reaction in reactions]
obj['submessages'] = submessages
return obj
def save_message_for_edit_use_case(message: Message) -> None:
message.save(update_fields=[
TOPIC_NAME, "content", "rendered_content", "rendered_content_version",
"last_edit_time", "edit_history", "has_attachment", "has_image",
"has_link", "recipient_id"
])
def build_message_dict(
message: Optional[Message], message_id: int,
last_edit_time: Optional[datetime.datetime],
edit_history: Optional[str], content: str, topic_name: str,
date_sent: datetime.datetime, rendered_content: Optional[str],
rendered_content_version: Optional[int], sender_id: int,
sender_realm_id: int, sending_client_name: str, recipient_id: int,
recipient_type: int, recipient_type_id: int,
reactions: List[Dict[str, Any]],
submessages: List[Dict[str, Any]]) -> Dict[str, Any]:
obj = dict(id=message_id,
sender_id=sender_id,
content=content,
recipient_type_id=recipient_type_id,
recipient_type=recipient_type,
recipient_id=recipient_id,
timestamp=datetime_to_timestamp(date_sent),
client=sending_client_name)
obj[TOPIC_NAME] = topic_name
obj['sender_realm_id'] = sender_realm_id
# Render topic_links with the stream's realm instead of the
# user's realm; this is important for messages sent by
# cross-realm bots like NOTIFICATION_BOT.
#
# TODO: We could potentially avoid this database query in
# common cases by optionally passing through the
# stream_realm_id through the code path from do_send_messages
# (where we've already fetched the data). It would involve
# somewhat messy plumbing, but would probably be worth it.
rendering_realm_id = sender_realm_id
if message and recipient_type == Recipient.STREAM:
rendering_realm_id = Stream.objects.get(
id=recipient_type_id).realm_id
obj[TOPIC_LINKS] = bugdown.topic_links(rendering_realm_id, topic_name)
if last_edit_time is not None:
obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
assert edit_history is not None
obj['edit_history'] = ujson.loads(edit_history)
if Message.need_to_render_content(rendered_content,
rendered_content_version,
bugdown.version):
if message is None:
# We really shouldn't be rendering objects in this method, but there is
# a scenario where we upgrade the version of bugdown and fail to run
# management commands to re-render historical messages, and then we
# need to have side effects. This method is optimized to not need full
# blown ORM objects, but the bugdown renderer is unfortunately highly
# coupled to Message, and we also need to persist the new rendered content.
# If we don't have a message object passed in, we get one here. The cost
# of going to the DB here should be overshadowed by the cost of rendering
# and updating the row.
# TODO: see #1379 to eliminate bugdown dependencies
message = Message.objects.select_related().get(id=message_id)
assert message is not None # Hint for mypy.
# It's unfortunate that we need to have side effects on the message
# in some cases.
rendered_content = save_message_rendered_content(message, content)
if rendered_content is not None:
obj['rendered_content'] = rendered_content
else:
obj['rendered_content'] = (
'<p>[Zulip note: Sorry, we could not ' +
'understand the formatting of your message]</p>')
if rendered_content is not None:
obj['is_me_message'] = Message.is_status_message(
content, rendered_content)
else:
obj['is_me_message'] = False
obj['reactions'] = [
ReactionDict.build_dict_from_raw_db_row(reaction)
for reaction in reactions
]
obj['submessages'] = submessages
return obj
def build_message_dict(
message: Optional[Message], message_id: int,
last_edit_time: Optional[datetime.datetime],
edit_history: Optional[str], content: str, subject: str,
pub_date: datetime.datetime, rendered_content: Optional[str],
rendered_content_version: Optional[int], sender_id: int,
sender_realm_id: int, sending_client_name: str, recipient_id: int,
recipient_type: int, recipient_type_id: int,
reactions: List[Dict[str, Any]],
submessages: List[Dict[str, Any]]) -> Dict[str, Any]:
obj = dict(id=message_id,
sender_id=sender_id,
content=content,
recipient_type_id=recipient_type_id,
recipient_type=recipient_type,
recipient_id=recipient_id,
subject=subject,
timestamp=datetime_to_timestamp(pub_date),
client=sending_client_name)
obj['sender_realm_id'] = sender_realm_id
obj['raw_display_recipient'] = get_display_recipient_by_id(
recipient_id, recipient_type, recipient_type_id)
obj['subject_links'] = bugdown.topic_links(sender_realm_id, subject)
if last_edit_time is not None:
obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
assert edit_history is not None
obj['edit_history'] = ujson.loads(edit_history)
if Message.need_to_render_content(rendered_content,
rendered_content_version,
bugdown.version):
if message is None:
# We really shouldn't be rendering objects in this method, but there is
# a scenario where we upgrade the version of bugdown and fail to run
# management commands to re-render historical messages, and then we
# need to have side effects. This method is optimized to not need full
# blown ORM objects, but the bugdown renderer is unfortunately highly
# coupled to Message, and we also need to persist the new rendered content.
# If we don't have a message object passed in, we get one here. The cost
# of going to the DB here should be overshadowed by the cost of rendering
# and updating the row.
# TODO: see #1379 to eliminate bugdown dependencies
message = Message.objects.select_related().get(id=message_id)
assert message is not None # Hint for mypy.
# It's unfortunate that we need to have side effects on the message
# in some cases.
rendered_content = save_message_rendered_content(message, content)
if rendered_content is not None:
obj['rendered_content'] = rendered_content
else:
obj['rendered_content'] = (
'<p>[Zulip note: Sorry, we could not ' +
'understand the formatting of your message]</p>')
if rendered_content is not None:
obj['is_me_message'] = Message.is_status_message(
content, rendered_content)
else:
obj['is_me_message'] = False
obj['reactions'] = [
ReactionDict.build_dict_from_raw_db_row(reaction)
for reaction in reactions
]
obj['submessages'] = submessages
return obj
def get_old_messages_backend(request, user_profile,
anchor = REQ(converter=int),
num_before = REQ(converter=to_non_negative_int),
num_after = REQ(converter=to_non_negative_int),
narrow = REQ('narrow', converter=narrow_parameter, default=None),
use_first_unread_anchor = REQ(default=False, converter=ujson.loads),
apply_markdown=REQ(default=True,
converter=ujson.loads)):
include_history = ok_to_include_history(narrow, user_profile.realm)
if include_history and not use_first_unread_anchor:
query = select([column("id").label("message_id")], None, "zerver_message")
inner_msg_id_col = literal_column("zerver_message.id")
elif narrow is None:
query = select([column("message_id"), column("flags")],
column("user_profile_id") == literal(user_profile.id),
"zerver_usermessage")
inner_msg_id_col = column("message_id")
else:
# TODO: Don't do this join if we're not doing a search
query = select([column("message_id"), column("flags")],
column("user_profile_id") == literal(user_profile.id),
join("zerver_usermessage", "zerver_message",
literal_column("zerver_usermessage.message_id") ==
literal_column("zerver_message.id")))
inner_msg_id_col = column("message_id")
num_extra_messages = 1
is_search = False
if narrow is not None:
# Add some metadata to our logging data for narrows
verbose_operators = []
for term in narrow:
if term['operator'] == "is":
verbose_operators.append("is:" + term['operand'])
else:
verbose_operators.append(term['operator'])
request._log_data['extra'] = "[%s]" % (",".join(verbose_operators),)
# Build the query for the narrow
num_extra_messages = 0
builder = NarrowBuilder(user_profile, inner_msg_id_col)
for term in narrow:
if term['operator'] == 'search' and not is_search:
query = query.column("subject").column("rendered_content")
is_search = True
query = builder.add_term(query, term)
# We add 1 to the number of messages requested if no narrow was
# specified to ensure that the resulting list always contains the
# anchor message. If a narrow was specified, the anchor message
# might not match the narrow anyway.
if num_after != 0:
num_after += num_extra_messages
else:
num_before += num_extra_messages
sa_conn = get_sqlalchemy_connection()
if use_first_unread_anchor:
condition = column("flags").op("&")(UserMessage.flags.read.mask) == 0
# We exclude messages on muted topics when finding the first unread
# message in this narrow
muting_conditions = exclude_muting_conditions(user_profile, narrow)
if muting_conditions:
condition = and_(condition, *muting_conditions)
first_unread_query = query.where(condition)
first_unread_query = first_unread_query.order_by(inner_msg_id_col.asc()).limit(1)
first_unread_result = list(sa_conn.execute(first_unread_query).fetchall())
if len(first_unread_result) > 0:
anchor = first_unread_result[0][0]
else:
anchor = 10000000000000000
before_query = None
after_query = None
if num_before != 0:
before_anchor = anchor
if num_after != 0:
# Don't include the anchor in both the before query and the after query
before_anchor = anchor - 1
before_query = query.where(inner_msg_id_col <= before_anchor) \
.order_by(inner_msg_id_col.desc()).limit(num_before)
if num_after != 0:
after_query = query.where(inner_msg_id_col >= anchor) \
.order_by(inner_msg_id_col.asc()).limit(num_after)
if num_before == 0 and num_after == 0:
# This can happen when a narrow is specified.
after_query = query.where(inner_msg_id_col == anchor)
if before_query is not None:
if after_query is not None:
query = union_all(before_query.self_group(), after_query.self_group())
else:
query = before_query
else:
query = after_query
main_query = alias(query)
query = select(main_query.c, None, main_query).order_by(column("message_id").asc())
# This is a hack to tag the query we use for testing
query = query.prefix_with("/* get_old_messages */")
query_result = list(sa_conn.execute(query).fetchall())
# The following is a little messy, but ensures that the code paths
# are similar regardless of the value of include_history. The
# 'user_messages' dictionary maps each message to the user's
# UserMessage object for that message, which we will attach to the
# rendered message dict before returning it. We attempt to
# bulk-fetch rendered message dicts from remote cache using the
# 'messages' list.
search_fields = dict()
message_ids = []
user_message_flags = {}
if include_history:
message_ids = [row[0] for row in query_result]
# TODO: This could be done with an outer join instead of two queries
user_message_flags = dict((user_message.message_id, user_message.flags_list()) for user_message in
UserMessage.objects.filter(user_profile=user_profile,
message__id__in=message_ids))
for row in query_result:
message_id = row[0]
if user_message_flags.get(message_id) is None:
user_message_flags[message_id] = ["read", "historical"]
if is_search:
(_, subject, rendered_content, content_matches, subject_matches) = row
search_fields[message_id] = get_search_fields(rendered_content, subject,
content_matches, subject_matches)
else:
for row in query_result:
message_id = row[0]
flags = row[1]
user_message_flags[message_id] = parse_usermessage_flags(flags)
message_ids.append(message_id)
if is_search:
(_, _, subject, rendered_content, content_matches, subject_matches) = row
search_fields[message_id] = get_search_fields(rendered_content, subject,
content_matches, subject_matches)
cache_transformer = lambda row: Message.build_dict_from_raw_db_row(row, apply_markdown)
id_fetcher = lambda row: row['id']
message_dicts = generic_bulk_cached_fetch(lambda message_id: to_dict_cache_key_id(message_id, apply_markdown),
Message.get_raw_db_rows,
message_ids,
id_fetcher=id_fetcher,
cache_transformer=cache_transformer,
extractor=extract_message_dict,
setter=stringify_message_dict)
message_list = []
for message_id in message_ids:
msg_dict = message_dicts[message_id]
msg_dict.update({"flags": user_message_flags[message_id]})
msg_dict.update(search_fields.get(message_id, {}))
message_list.append(msg_dict)
statsd.incr('loaded_old_messages', len(message_list))
ret = {'messages': message_list,
"result": "success",
"msg": ""}
return json_success(ret)
def do_test_subject(self, msg: Message, expected_subject: Optional[Text]) -> None:
if expected_subject is not None:
self.assertEqual(msg.topic_name(), expected_subject)
def generate_and_send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open(
os.path.join(get_or_create_dev_uuid_var_path('test-backend'),
"test_messages.json"), "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
] # type: List[int]
recipient_huddles = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
message_batch_size = options['batch_size']
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
messages = []
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100. / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + str(random.randint(1, 3))
saved_data['subject'] = message.subject
message.date_sent = choose_date_sent(num_messages, tot_messages,
options['threads'])
messages.append(message)
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
if (num_messages % message_batch_size) == 0:
# Send the batch and empty the list:
send_messages(messages)
messages = []
if len(messages) > 0:
# If there are unsent messages after exiting the loop, send them:
send_messages(messages)
return tot_messages
def build_message_dict(
apply_markdown,
message,
message_id,
last_edit_time,
edit_history,
content,
subject,
pub_date,
rendered_content,
rendered_content_version,
sender_id,
sender_email,
sender_realm_domain,
sender_full_name,
sender_short_name,
sender_avatar_source,
sender_is_mirror_dummy,
sending_client_name,
recipient_id,
recipient_type,
recipient_type_id,
):
# type: (bool, Message, int, datetime.datetime, text_type, text_type, text_type, datetime.datetime, text_type, Optional[int], int, text_type, text_type, text_type, text_type, text_type, bool, text_type, int, int, int) -> Dict[str, Any]
avatar_url = get_avatar_url(sender_avatar_source, sender_email)
display_recipient = get_display_recipient_by_id(
recipient_id,
recipient_type,
recipient_type_id
)
if recipient_type == Recipient.STREAM:
display_type = "stream"
elif recipient_type in (Recipient.HUDDLE, Recipient.PERSONAL):
assert not isinstance(display_recipient, text_type)
display_type = "private"
if len(display_recipient) == 1:
# add the sender in if this isn't a message between
# someone and his self, preserving ordering
recip = {'email': sender_email,
'domain': sender_realm_domain,
'full_name': sender_full_name,
'short_name': sender_short_name,
'id': sender_id,
'is_mirror_dummy': sender_is_mirror_dummy}
if recip['email'] < display_recipient[0]['email']:
display_recipient = [recip, display_recipient[0]]
elif recip['email'] > display_recipient[0]['email']:
display_recipient = [display_recipient[0], recip]
obj = dict(
id = message_id,
sender_email = sender_email,
sender_full_name = sender_full_name,
sender_short_name = sender_short_name,
sender_domain = sender_realm_domain,
sender_id = sender_id,
type = display_type,
display_recipient = display_recipient,
recipient_id = recipient_id,
subject = subject,
timestamp = datetime_to_timestamp(pub_date),
gravatar_hash = gravatar_hash(sender_email), # Deprecated June 2013
avatar_url = avatar_url,
client = sending_client_name)
obj['subject_links'] = bugdown.subject_links(sender_realm_domain.lower(), subject)
if last_edit_time != None:
obj['last_edit_timestamp'] = datetime_to_timestamp(last_edit_time)
obj['edit_history'] = ujson.loads(edit_history)
if apply_markdown:
if Message.need_to_render_content(rendered_content, rendered_content_version, bugdown.version):
if message is None:
# We really shouldn't be rendering objects in this method, but there is
# a scenario where we upgrade the version of bugdown and fail to run
# management commands to re-render historical messages, and then we
# need to have side effects. This method is optimized to not need full
# blown ORM objects, but the bugdown renderer is unfortunately highly
# coupled to Message, and we also need to persist the new rendered content.
# If we don't have a message object passed in, we get one here. The cost
# of going to the DB here should be overshadowed by the cost of rendering
# and updating the row.
# TODO: see #1379 to eliminate bugdown dependencies
message = Message.objects.select_related().get(id=message_id)
# It's unfortunate that we need to have side effects on the message
# in some cases.
rendered_content = render_markdown(message, content, sender_realm_domain)
message.rendered_content = rendered_content
message.rendered_content_version = bugdown.version
message.save_rendered_content()
if rendered_content is not None:
obj['content'] = rendered_content
else:
obj['content'] = u'<p>[Zulip note: Sorry, we could not understand the formatting of your message]</p>'
obj['content_type'] = 'text/html'
else:
obj['content'] = content
obj['content_type'] = 'text/x-markdown'
return obj
def fill_edit_history_entries(
raw_edit_history: List[EditHistoryEvent],
message: Message) -> List[FormattedEditHistoryEvent]:
"""
This fills out the message edit history entries from the database
to have the current topic + content as of that time, plus data on
whatever changed. This makes it much simpler to do future
processing.
"""
prev_content = message.content
prev_rendered_content = message.rendered_content
prev_topic = message.topic_name()
# Make sure that the latest entry in the history corresponds to the
# message's last edit time
if len(raw_edit_history) > 0:
assert message.last_edit_time is not None
assert datetime_to_timestamp(
message.last_edit_time) == raw_edit_history[0]["timestamp"]
formatted_edit_history: List[FormattedEditHistoryEvent] = []
for edit_history_event in raw_edit_history:
formatted_entry: FormattedEditHistoryEvent = {
"content": prev_content,
"rendered_content": prev_rendered_content,
"timestamp": edit_history_event["timestamp"],
"topic": prev_topic,
"user_id": edit_history_event["user_id"],
}
if "prev_topic" in edit_history_event:
prev_topic = edit_history_event["prev_topic"]
formatted_entry["prev_topic"] = prev_topic
# Fill current values for content/rendered_content.
if "prev_content" in edit_history_event:
formatted_entry["prev_content"] = edit_history_event[
"prev_content"]
prev_content = formatted_entry["prev_content"]
formatted_entry["prev_rendered_content"] = edit_history_event[
"prev_rendered_content"]
prev_rendered_content = formatted_entry["prev_rendered_content"]
assert prev_rendered_content is not None
rendered_content = formatted_entry["rendered_content"]
assert rendered_content is not None
formatted_entry["content_html_diff"] = highlight_html_differences(
prev_rendered_content, rendered_content, message.id)
if "prev_stream" in edit_history_event:
formatted_entry["prev_stream"] = edit_history_event["prev_stream"]
formatted_entry["stream"] = edit_history_event["stream"]
formatted_edit_history.append(formatted_entry)
initial_message_history: FormattedEditHistoryEvent = {
"content": prev_content,
"rendered_content": prev_rendered_content,
"timestamp": datetime_to_timestamp(message.date_sent),
"topic": prev_topic,
"user_id": message.sender_id,
}
formatted_edit_history.append(initial_message_history)
return formatted_edit_history
def generate_and_send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open(
os.path.join(get_or_create_dev_uuid_var_path("test-backend"),
"test_messages.json"), "rb") as infile:
dialog = orjson.loads(infile.read())
random.shuffle(dialog)
texts = itertools.cycle(dialog)
# We need to filter out streams from the analytics realm as we don't want to generate
# messages to its streams - and they might also have no subscribers, which would break
# our message generation mechanism below.
stream_ids = Stream.objects.filter(realm=get_realm("zulip")).values_list(
"id", flat=True)
recipient_streams: List[int] = [
recipient.id
for recipient in Recipient.objects.filter(type=Recipient.STREAM,
type_id__in=stream_ids)
]
recipient_huddles: List[int] = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
]
huddle_members: Dict[int, List[int]] = {}
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
# Generate different topics for each stream
possible_topics = {}
for stream_id in recipient_streams:
possible_topics[stream_id] = generate_topics(options["max_topics"])
message_batch_size = options["batch_size"]
num_messages = 0
random_max = 1000000
recipients: Dict[int, Tuple[int, int, Dict[str, Any]]] = {}
messages: List[Message] = []
while num_messages < tot_messages:
saved_data: Dict[str, Any] = {}
message = Message()
message.sending_client = get_client("populate_db")
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100.0 / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data["personals_pair"]
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data["subject"]
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif randkey <= random_max * options["percent_huddles"] / 100.0:
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.0):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif randkey <= random_max * 1.0:
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data["personals_pair"] = personals_pair
elif message_type == Recipient.STREAM:
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = random.choice(
possible_topics[message.recipient.id])
saved_data["subject"] = message.subject
message.date_sent = choose_date_sent(num_messages, tot_messages,
options["threads"])
messages.append(message)
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
if (num_messages % message_batch_size) == 0:
# Send the batch and empty the list:
send_messages(messages)
messages = []
if len(messages) > 0:
# If there are unsent messages after exiting the loop, send them:
send_messages(messages)
return tot_messages
def build_message_dict(
message_id: int,
last_edit_time: Optional[datetime.datetime],
edit_history: Optional[str],
content: str,
topic_name: str,
date_sent: datetime.datetime,
rendered_content: Optional[str],
rendered_content_version: Optional[int],
sender_id: int,
sender_realm_id: int,
sending_client_name: str,
rendering_realm_id: int,
recipient_id: int,
recipient_type: int,
recipient_type_id: int,
reactions: List[RawReactionRow],
submessages: List[Dict[str, Any]],
) -> Dict[str, Any]:
obj = dict(
id=message_id,
sender_id=sender_id,
content=content,
recipient_type_id=recipient_type_id,
recipient_type=recipient_type,
recipient_id=recipient_id,
timestamp=datetime_to_timestamp(date_sent),
client=sending_client_name,
)
obj[TOPIC_NAME] = topic_name
obj["sender_realm_id"] = sender_realm_id
# Render topic_links with the stream's realm instead of the
# sender's realm; this is important for messages sent by
# cross-realm bots like NOTIFICATION_BOT.
obj[TOPIC_LINKS] = topic_links(rendering_realm_id, topic_name)
if last_edit_time is not None:
obj["last_edit_timestamp"] = datetime_to_timestamp(last_edit_time)
assert edit_history is not None
obj["edit_history"] = orjson.loads(edit_history)
if Message.need_to_render_content(rendered_content,
rendered_content_version,
markdown_version):
# We really shouldn't be rendering objects in this method, but there is
# a scenario where we upgrade the version of Markdown and fail to run
# management commands to re-render historical messages, and then we
# need to have side effects. This method is optimized to not need full
# blown ORM objects, but the Markdown renderer is unfortunately highly
# coupled to Message, and we also need to persist the new rendered content.
# If we don't have a message object passed in, we get one here. The cost
# of going to the DB here should be overshadowed by the cost of rendering
# and updating the row.
# TODO: see #1379 to eliminate Markdown dependencies
message = Message.objects.select_related().get(id=message_id)
assert message is not None # Hint for mypy.
# It's unfortunate that we need to have side effects on the message
# in some cases.
rendered_content = save_message_rendered_content(message, content)
if rendered_content is not None:
obj["rendered_content"] = rendered_content
else:
obj["rendered_content"] = (
"<p>[Zulip note: Sorry, we could not " +
"understand the formatting of your message]</p>")
if rendered_content is not None:
obj["is_me_message"] = Message.is_status_message(
content, rendered_content)
else:
obj["is_me_message"] = False
obj["reactions"] = [
ReactionDict.build_dict_from_raw_db_row(reaction)
for reaction in reactions
]
obj["submessages"] = submessages
return obj
def do_test_topic(self, msg: Message, expected_topic: Optional[str]) -> None:
if expected_topic is not None:
self.assertEqual(msg.topic_name(), expected_topic)
def get_old_messages_backend(request, user_profile,
anchor = REQ(converter=int),
num_before = REQ(converter=to_non_negative_int),
num_after = REQ(converter=to_non_negative_int),
narrow = REQ('narrow', converter=narrow_parameter, default=None),
use_first_unread_anchor = REQ(default=False, converter=ujson.loads),
apply_markdown=REQ(default=True,
converter=ujson.loads)):
include_history = ok_to_include_history(narrow, user_profile.realm)
if include_history and not use_first_unread_anchor:
query = select([column("id").label("message_id")], None, "zerver_message")
inner_msg_id_col = literal_column("zerver_message.id")
elif narrow is None:
query = select([column("message_id"), column("flags")],
column("user_profile_id") == literal(user_profile.id),
"zerver_usermessage")
inner_msg_id_col = column("message_id")
else:
# TODO: Don't do this join if we're not doing a search
query = select([column("message_id"), column("flags")],
column("user_profile_id") == literal(user_profile.id),
join("zerver_usermessage", "zerver_message",
literal_column("zerver_usermessage.message_id") ==
literal_column("zerver_message.id")))
inner_msg_id_col = column("message_id")
num_extra_messages = 1
is_search = False
if narrow is not None:
# Add some metadata to our logging data for narrows
verbose_operators = []
for term in narrow:
if term['operator'] == "is":
verbose_operators.append("is:" + term['operand'])
else:
verbose_operators.append(term['operator'])
request._log_data['extra'] = "[%s]" % (",".join(verbose_operators),)
# Build the query for the narrow
num_extra_messages = 0
builder = NarrowBuilder(user_profile, inner_msg_id_col)
for term in narrow:
if term['operator'] == 'search' and not is_search:
query = query.column("subject").column("rendered_content")
is_search = True
query = builder.add_term(query, term)
# We add 1 to the number of messages requested if no narrow was
# specified to ensure that the resulting list always contains the
# anchor message. If a narrow was specified, the anchor message
# might not match the narrow anyway.
if num_after != 0:
num_after += num_extra_messages
else:
num_before += num_extra_messages
sa_conn = get_sqlalchemy_connection()
if use_first_unread_anchor:
condition = column("flags").op("&")(UserMessage.flags.read.mask) == 0
# We exclude messages on muted topics when finding the first unread
# message in this narrow
muting_conditions = exclude_muting_conditions(user_profile, narrow)
if muting_conditions:
condition = and_(condition, *muting_conditions)
first_unread_query = query.where(condition)
first_unread_query = first_unread_query.order_by(inner_msg_id_col.asc()).limit(1)
first_unread_result = list(sa_conn.execute(first_unread_query).fetchall())
if len(first_unread_result) > 0:
anchor = first_unread_result[0][0]
else:
anchor = 10000000000000000
before_query = None
after_query = None
if num_before != 0:
before_anchor = anchor
if num_after != 0:
# Don't include the anchor in both the before query and the after query
before_anchor = anchor - 1
before_query = query.where(inner_msg_id_col <= before_anchor) \
.order_by(inner_msg_id_col.desc()).limit(num_before)
if num_after != 0:
after_query = query.where(inner_msg_id_col >= anchor) \
.order_by(inner_msg_id_col.asc()).limit(num_after)
if num_before == 0 and num_after == 0:
# This can happen when a narrow is specified.
after_query = query.where(inner_msg_id_col == anchor)
if before_query is not None:
if after_query is not None:
query = union_all(before_query.self_group(), after_query.self_group())
else:
query = before_query
else:
query = after_query
main_query = alias(query)
query = select(main_query.c, None, main_query).order_by(column("message_id").asc())
# This is a hack to tag the query we use for testing
query = query.prefix_with("/* get_old_messages */")
query_result = list(sa_conn.execute(query).fetchall())
# The following is a little messy, but ensures that the code paths
# are similar regardless of the value of include_history. The
# 'user_messages' dictionary maps each message to the user's
# UserMessage object for that message, which we will attach to the
# rendered message dict before returning it. We attempt to
# bulk-fetch rendered message dicts from remote cache using the
# 'messages' list.
search_fields = dict() # type: Dict[int, Dict[str, str]]
message_ids = [] # type: List[int]
user_message_flags = {} # type: Dict[int, List[str]]
if include_history:
message_ids = [row[0] for row in query_result]
# TODO: This could be done with an outer join instead of two queries
user_message_flags = dict((user_message.message_id, user_message.flags_list()) for user_message in
UserMessage.objects.filter(user_profile=user_profile,
message__id__in=message_ids))
for row in query_result:
message_id = row[0]
if user_message_flags.get(message_id) is None:
user_message_flags[message_id] = ["read", "historical"]
if is_search:
(_, subject, rendered_content, content_matches, subject_matches) = row
search_fields[message_id] = get_search_fields(rendered_content, subject,
content_matches, subject_matches)
else:
for row in query_result:
message_id = row[0]
flags = row[1]
user_message_flags[message_id] = parse_usermessage_flags(flags)
message_ids.append(message_id)
if is_search:
(_, _, subject, rendered_content, content_matches, subject_matches) = row
search_fields[message_id] = get_search_fields(rendered_content, subject,
content_matches, subject_matches)
cache_transformer = lambda row: Message.build_dict_from_raw_db_row(row, apply_markdown)
id_fetcher = lambda row: row['id']
message_dicts = generic_bulk_cached_fetch(lambda message_id: to_dict_cache_key_id(message_id, apply_markdown),
Message.get_raw_db_rows,
message_ids,
id_fetcher=id_fetcher,
cache_transformer=cache_transformer,
extractor=extract_message_dict,
setter=stringify_message_dict)
message_list = []
for message_id in message_ids:
msg_dict = message_dicts[message_id]
msg_dict.update({"flags": user_message_flags[message_id]})
msg_dict.update(search_fields.get(message_id, {}))
message_list.append(msg_dict)
statsd.incr('loaded_old_messages', len(message_list))
ret = {'messages': message_list,
"result": "success",
"msg": ""}
return json_success(ret)
def save_message_for_edit_use_case(message: Message) -> None:
message.save(update_fields=[
"subject", "content", "rendered_content", "rendered_content_version",
"last_edit_time", "edit_history"
])
def send_messages(data):
# type: (Tuple[int, Sequence[Sequence[int]], Mapping[str, Any], Callable[[str], Any], int]) -> int
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open("var/test_messages.json", "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
] # type: List[int]
recipient_huddles = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100. / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + Text(random.randint(1, 3))
saved_data['subject'] = message.subject
message.pub_date = timezone_now()
do_send_messages([{'message': message}])
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
return tot_messages
def send_messages(
data: Tuple[int, Sequence[Sequence[int]], Mapping[str, Any],
Callable[[str], Any], int]
) -> int:
(tot_messages, personals_pairs, options, output, random_seed) = data
random.seed(random_seed)
with open("var/test_messages.json", "r") as infile:
dialog = ujson.load(infile)
random.shuffle(dialog)
texts = itertools.cycle(dialog)
recipient_streams = [
klass.id for klass in Recipient.objects.filter(type=Recipient.STREAM)
] # type: List[int]
recipient_huddles = [
h.id for h in Recipient.objects.filter(type=Recipient.HUDDLE)
] # type: List[int]
huddle_members = {} # type: Dict[int, List[int]]
for h in recipient_huddles:
huddle_members[h] = [
s.user_profile.id
for s in Subscription.objects.filter(recipient_id=h)
]
num_messages = 0
random_max = 1000000
recipients = {} # type: Dict[int, Tuple[int, int, Dict[str, Any]]]
while num_messages < tot_messages:
saved_data = {} # type: Dict[str, Any]
message = Message()
message.sending_client = get_client('populate_db')
message.content = next(texts)
randkey = random.randint(1, random_max)
if (num_messages > 0
and random.randint(1, random_max) * 100. / random_max <
options["stickyness"]):
# Use an old recipient
message_type, recipient_id, saved_data = recipients[num_messages -
1]
if message_type == Recipient.PERSONAL:
personals_pair = saved_data['personals_pair']
random.shuffle(personals_pair)
elif message_type == Recipient.STREAM:
message.subject = saved_data['subject']
message.recipient = get_recipient_by_id(recipient_id)
elif message_type == Recipient.HUDDLE:
message.recipient = get_recipient_by_id(recipient_id)
elif (randkey <= random_max * options["percent_huddles"] / 100.):
message_type = Recipient.HUDDLE
message.recipient = get_recipient_by_id(
random.choice(recipient_huddles))
elif (randkey <= random_max *
(options["percent_huddles"] + options["percent_personals"]) /
100.):
message_type = Recipient.PERSONAL
personals_pair = random.choice(personals_pairs)
random.shuffle(personals_pair)
elif (randkey <= random_max * 1.0):
message_type = Recipient.STREAM
message.recipient = get_recipient_by_id(
random.choice(recipient_streams))
if message_type == Recipient.HUDDLE:
sender_id = random.choice(huddle_members[message.recipient.id])
message.sender = get_user_profile_by_id(sender_id)
elif message_type == Recipient.PERSONAL:
message.recipient = Recipient.objects.get(
type=Recipient.PERSONAL, type_id=personals_pair[0])
message.sender = get_user_profile_by_id(personals_pair[1])
saved_data['personals_pair'] = personals_pair
elif message_type == Recipient.STREAM:
stream = Stream.objects.get(id=message.recipient.type_id)
# Pick a random subscriber to the stream
message.sender = random.choice(
Subscription.objects.filter(
recipient=message.recipient)).user_profile
message.subject = stream.name + str(random.randint(1, 3))
saved_data['subject'] = message.subject
# Spoofing time not supported with threading
if options['threads'] != 1:
message.pub_date = timezone_now()
else:
# Distrubutes 80% of messages starting from 5 days ago, over a period
# of 3 days. Then, distributes remaining messages over past 24 hours.
spoofed_date = timezone_now() - timezone_timedelta(days=5)
if (num_messages < tot_messages * 0.8):
# Maximum of 3 days ahead, convert to minutes
time_ahead = 3 * 24 * 60
time_ahead //= int(tot_messages * 0.8)
else:
time_ahead = 24 * 60
time_ahead //= int(tot_messages * 0.2)
spoofed_minute = random.randint(time_ahead * num_messages,
time_ahead * (num_messages + 1))
spoofed_date += timezone_timedelta(minutes=spoofed_minute)
message.pub_date = spoofed_date
# We disable USING_RABBITMQ here, so that deferred work is
# executed in do_send_message_messages, rather than being
# queued. This is important, because otherwise, if run-dev.py
# wasn't running when populate_db was run, a developer can end
# up with queued events that reference objects from a previous
# life of the database, which naturally throws exceptions.
settings.USING_RABBITMQ = False
do_send_messages([{'message': message}])
settings.USING_RABBITMQ = True
recipients[num_messages] = (message_type, message.recipient.id,
saved_data)
num_messages += 1
return tot_messages
def do_update_message(
user_profile: UserProfile,
target_message: Message,
new_stream: Optional[Stream],
topic_name: Optional[str],
propagate_mode: Optional[str],
send_notification_to_old_thread: bool,
send_notification_to_new_thread: bool,
content: Optional[str],
rendering_result: Optional[MessageRenderingResult],
prior_mention_user_ids: Set[int],
mention_data: Optional[MentionData] = None,
) -> int:
"""
The main function for message editing. A message edit event can
modify:
* the message's content (in which case the caller will have
set both content and rendered_content),
* the topic, in which case the caller will have set topic_name
* or both message's content and the topic
* or stream and/or topic, in which case the caller will have set
new_stream and/or topic_name.
With topic edits, propagate_mode determines whether other message
also have their topics edited.
"""
timestamp = timezone_now()
target_message.last_edit_time = timestamp
event: Dict[str, Any] = {
"type": "update_message",
"user_id": user_profile.id,
"edit_timestamp": datetime_to_timestamp(timestamp),
"message_id": target_message.id,
"rendering_only": False,
}
edit_history_event: EditHistoryEvent = {
"user_id": user_profile.id,
"timestamp": event["edit_timestamp"],
}
changed_messages = [target_message]
realm = user_profile.realm
stream_being_edited = None
if target_message.is_stream_message():
stream_id = target_message.recipient.type_id
stream_being_edited = get_stream_by_id_in_realm(stream_id, realm)
event["stream_name"] = stream_being_edited.name
event["stream_id"] = stream_being_edited.id
ums = UserMessage.objects.filter(message=target_message.id)
if content is not None:
assert rendering_result is not None
# mention_data is required if there's a content edit.
assert mention_data is not None
# add data from group mentions to mentions_user_ids.
for group_id in rendering_result.mentions_user_group_ids:
members = mention_data.get_group_members(group_id)
rendering_result.mentions_user_ids.update(members)
update_user_message_flags(rendering_result, ums)
# One could imagine checking realm.allow_edit_history here and
# modifying the events based on that setting, but doing so
# doesn't really make sense. We need to send the edit event
# to clients regardless, and a client already had access to
# the original/pre-edit content of the message anyway. That
# setting must be enforced on the client side, and making a
# change here simply complicates the logic for clients parsing
# edit history events.
event["orig_content"] = target_message.content
event["orig_rendered_content"] = target_message.rendered_content
edit_history_event["prev_content"] = target_message.content
edit_history_event[
"prev_rendered_content"] = target_message.rendered_content
edit_history_event[
"prev_rendered_content_version"] = target_message.rendered_content_version
target_message.content = content
target_message.rendered_content = rendering_result.rendered_content
target_message.rendered_content_version = markdown_version
event["content"] = content
event["rendered_content"] = rendering_result.rendered_content
event[
"prev_rendered_content_version"] = target_message.rendered_content_version
event["is_me_message"] = Message.is_status_message(
content, rendering_result.rendered_content)
# target_message.has_image and target_message.has_link will have been
# already updated by Markdown rendering in the caller.
target_message.has_attachment = check_attachment_reference_change(
target_message, rendering_result)
if target_message.is_stream_message():
if topic_name is not None:
new_topic_name = topic_name
else:
new_topic_name = target_message.topic_name()
stream_topic: Optional[StreamTopicTarget] = StreamTopicTarget(
stream_id=stream_id,
topic_name=new_topic_name,
)
else:
stream_topic = None
info = get_recipient_info(
realm_id=realm.id,
recipient=target_message.recipient,
sender_id=target_message.sender_id,
stream_topic=stream_topic,
possible_wildcard_mention=mention_data.message_has_wildcards(),
)
event["online_push_user_ids"] = list(info["online_push_user_ids"])
event["pm_mention_push_disabled_user_ids"] = list(
info["pm_mention_push_disabled_user_ids"])
event["pm_mention_email_disabled_user_ids"] = list(
info["pm_mention_email_disabled_user_ids"])
event["stream_push_user_ids"] = list(info["stream_push_user_ids"])
event["stream_email_user_ids"] = list(info["stream_email_user_ids"])
event["muted_sender_user_ids"] = list(info["muted_sender_user_ids"])
event["prior_mention_user_ids"] = list(prior_mention_user_ids)
event["presence_idle_user_ids"] = filter_presence_idle_user_ids(
info["active_user_ids"])
event["all_bot_user_ids"] = list(info["all_bot_user_ids"])
if rendering_result.mentions_wildcard:
event["wildcard_mention_user_ids"] = list(
info["wildcard_mention_user_ids"])
else:
event["wildcard_mention_user_ids"] = []
do_update_mobile_push_notification(
target_message,
prior_mention_user_ids,
rendering_result.mentions_user_ids,
info["stream_push_user_ids"],
)
if topic_name is not None or new_stream is not None:
assert propagate_mode is not None
orig_topic_name = target_message.topic_name()
event["propagate_mode"] = propagate_mode
if new_stream is not None:
assert content is None
assert target_message.is_stream_message()
assert stream_being_edited is not None
edit_history_event["prev_stream"] = stream_being_edited.id
edit_history_event["stream"] = new_stream.id
event[ORIG_TOPIC] = orig_topic_name
assert new_stream.recipient_id is not None
target_message.recipient_id = new_stream.recipient_id
event["new_stream_id"] = new_stream.id
event["propagate_mode"] = propagate_mode
# When messages are moved from one stream to another, some
# users may lose access to those messages, including guest
# users and users not subscribed to the new stream (if it is a
# private stream). For those users, their experience is as
# though the messages were deleted, and we should send a
# delete_message event to them instead.
subs_to_old_stream = get_active_subscriptions_for_stream_id(
stream_id,
include_deactivated_users=True).select_related("user_profile")
subs_to_new_stream = list(
get_active_subscriptions_for_stream_id(
new_stream.id,
include_deactivated_users=True).select_related("user_profile"))
old_stream_sub_ids = [
user.user_profile_id for user in subs_to_old_stream
]
new_stream_sub_ids = [
user.user_profile_id for user in subs_to_new_stream
]
# Get users who aren't subscribed to the new_stream.
subs_losing_usermessages = [
sub for sub in subs_to_old_stream
if sub.user_profile_id not in new_stream_sub_ids
]
# Users who can longer access the message without some action
# from administrators.
subs_losing_access = [
sub for sub in subs_losing_usermessages
if sub.user_profile.is_guest or not new_stream.is_public()
]
ums = ums.exclude(user_profile_id__in=[
sub.user_profile_id for sub in subs_losing_usermessages
])
subs_gaining_usermessages = []
if not new_stream.is_history_public_to_subscribers():
# For private streams, with history not public to subscribers,
# We find out users who are not present in the msgs' old stream
# and create new UserMessage for these users so that they can
# access this message.
subs_gaining_usermessages += [
user_id for user_id in new_stream_sub_ids
if user_id not in old_stream_sub_ids
]
if topic_name is not None:
topic_name = truncate_topic(topic_name)
target_message.set_topic_name(topic_name)
# These fields have legacy field names.
event[ORIG_TOPIC] = orig_topic_name
event[TOPIC_NAME] = topic_name
event[TOPIC_LINKS] = topic_links(target_message.sender.realm_id,
topic_name)
edit_history_event["prev_topic"] = orig_topic_name
edit_history_event["topic"] = topic_name
update_edit_history(target_message, timestamp, edit_history_event)
delete_event_notify_user_ids: List[int] = []
if propagate_mode in ["change_later", "change_all"]:
assert topic_name is not None or new_stream is not None
assert stream_being_edited is not None
# Other messages should only get topic/stream fields in their edit history.
topic_only_edit_history_event: EditHistoryEvent = {
"user_id": edit_history_event["user_id"],
"timestamp": edit_history_event["timestamp"],
}
if topic_name is not None:
topic_only_edit_history_event["prev_topic"] = edit_history_event[
"prev_topic"]
topic_only_edit_history_event["topic"] = edit_history_event[
"topic"]
if new_stream is not None:
topic_only_edit_history_event["prev_stream"] = edit_history_event[
"prev_stream"]
topic_only_edit_history_event["stream"] = edit_history_event[
"stream"]
messages_list = update_messages_for_topic_edit(
acting_user=user_profile,
edited_message=target_message,
propagate_mode=propagate_mode,
orig_topic_name=orig_topic_name,
topic_name=topic_name,
new_stream=new_stream,
old_stream=stream_being_edited,
edit_history_event=topic_only_edit_history_event,
last_edit_time=timestamp,
)
changed_messages += messages_list
if new_stream is not None:
assert stream_being_edited is not None
changed_message_ids = [msg.id for msg in changed_messages]
if subs_gaining_usermessages:
ums_to_create = []
for message_id in changed_message_ids:
for user_profile_id in subs_gaining_usermessages:
# The fact that the user didn't have a UserMessage originally means we can infer that the user
# was not mentioned in the original message (even if mention syntax was present, it would not
# take effect for a user who was not subscribed). If we were editing the message's content, we
# would rerender the message and then use the new stream's data to determine whether this is
# a mention of a subscriber; but as we are not doing so, we choose to preserve the "was this
# mention syntax an actual mention" decision made during the original rendering for implementation
# simplicity. As a result, the only flag to consider applying here is read.
um = UserMessageLite(
user_profile_id=user_profile_id,
message_id=message_id,
flags=UserMessage.flags.read,
)
ums_to_create.append(um)
bulk_insert_ums(ums_to_create)
# Delete UserMessage objects for users who will no
# longer have access to these messages. Note: This could be
# very expensive, since it's N guest users x M messages.
UserMessage.objects.filter(
user_profile_id__in=[
sub.user_profile_id for sub in subs_losing_usermessages
],
message_id__in=changed_message_ids,
).delete()
delete_event: DeleteMessagesEvent = {
"type": "delete_message",
"message_ids": changed_message_ids,
"message_type": "stream",
"stream_id": stream_being_edited.id,
"topic": orig_topic_name,
}
delete_event_notify_user_ids = [
sub.user_profile_id for sub in subs_losing_access
]
send_event(user_profile.realm, delete_event,
delete_event_notify_user_ids)
# Reset the Attachment.is_*_public caches for all messages
# moved to another stream with different access permissions.
if new_stream.invite_only != stream_being_edited.invite_only:
Attachment.objects.filter(
messages__in=changed_message_ids).update(
is_realm_public=None, )
ArchivedAttachment.objects.filter(
messages__in=changed_message_ids).update(
is_realm_public=None, )
if new_stream.is_web_public != stream_being_edited.is_web_public:
Attachment.objects.filter(
messages__in=changed_message_ids).update(
is_web_public=None, )
ArchivedAttachment.objects.filter(
messages__in=changed_message_ids).update(
is_web_public=None, )
# This does message.save(update_fields=[...])
save_message_for_edit_use_case(message=target_message)
realm_id: Optional[int] = None
if stream_being_edited is not None:
realm_id = stream_being_edited.realm_id
event["message_ids"] = update_to_dict_cache(changed_messages, realm_id)
def user_info(um: UserMessage) -> Dict[str, Any]:
return {
"id": um.user_profile_id,
"flags": um.flags_list(),
}
# The following blocks arranges that users who are subscribed to a
# stream and can see history from before they subscribed get
# live-update when old messages are edited (e.g. if the user does
# a topic edit themself).
#
# We still don't send an update event to users who are not
# subscribed to this stream and don't have a UserMessage row. This
# means if a non-subscriber is viewing the narrow, they won't get
# a real-time updates. This is a balance between sending
# message-edit notifications for every public stream to every user
# in the organization (too expansive, and also not what we do for
# newly sent messages anyway) and having magical live-updates
# where possible.
users_to_be_notified = list(map(user_info, ums))
if stream_being_edited is not None:
if stream_being_edited.is_history_public_to_subscribers():
subscriptions = get_active_subscriptions_for_stream_id(
stream_id, include_deactivated_users=False)
# We exclude long-term idle users, since they by
# definition have no active clients.
subscriptions = subscriptions.exclude(
user_profile__long_term_idle=True)
# Remove duplicates by excluding the id of users already
# in users_to_be_notified list. This is the case where a
# user both has a UserMessage row and is a current
# Subscriber
subscriptions = subscriptions.exclude(
user_profile_id__in=[um.user_profile_id for um in ums])
if new_stream is not None:
assert delete_event_notify_user_ids is not None
subscriptions = subscriptions.exclude(
user_profile_id__in=delete_event_notify_user_ids)
# All users that are subscribed to the stream must be
# notified when a message is edited
subscriber_ids = set(
subscriptions.values_list("user_profile_id", flat=True))
if new_stream is not None:
# TODO: Guest users don't see the new moved topic
# unless breadcrumb message for new stream is
# enabled. Excluding these users from receiving this
# event helps us avoid a error traceback for our
# clients. We should figure out a way to inform the
# guest users of this new topic if sending a 'message'
# event for these messages is not an option.
#
# Don't send this event to guest subs who are not
# subscribers of the old stream but are subscribed to
# the new stream; clients will be confused.
old_stream_unsubbed_guests = [
sub for sub in subs_to_new_stream
if sub.user_profile.is_guest
and sub.user_profile_id not in subscriber_ids
]
subscriptions = subscriptions.exclude(user_profile_id__in=[
sub.user_profile_id for sub in old_stream_unsubbed_guests
])
subscriber_ids = set(
subscriptions.values_list("user_profile_id", flat=True))
users_to_be_notified += list(
map(subscriber_info, sorted(list(subscriber_ids))))
# UserTopic updates and the content of notifications depend on
# whether we've moved the entire topic, or just part of it. We
# make that determination here.
moved_all_visible_messages = False
if topic_name is not None or new_stream is not None:
assert stream_being_edited is not None
if propagate_mode == "change_all":
moved_all_visible_messages = True
else:
# With other propagate modes, if the user in fact moved
# all messages in the stream, we want to explain it was a
# full-topic move.
#
# For security model reasons, we don't want to allow a
# user to take any action that would leak information
# about older messages they cannot access (E.g. the only
# remaining messages are in a stream without shared
# history). The bulk_access_messages call below addresses
# that concern.
#
# bulk_access_messages is inefficient for this task, since
# we just want to do the exists() version of this
# query. But it's nice to reuse code, and this bulk
# operation is likely cheaper than a `GET /messages`
# unless the topic has thousands of messages of history.
assert stream_being_edited.recipient_id is not None
unmoved_messages = messages_for_topic(
stream_being_edited.recipient_id,
orig_topic_name,
)
visible_unmoved_messages = bulk_access_messages(
user_profile, unmoved_messages, stream=stream_being_edited)
moved_all_visible_messages = len(visible_unmoved_messages) == 0
# Migrate muted topic configuration in the following circumstances:
#
# * If propagate_mode is change_all, do so unconditionally.
#
# * If propagate_mode is change_later or change_one, do so when
# the acting user has moved the entire topic (as visible to them).
#
# This rule corresponds to checking moved_all_visible_messages.
#
# We may want more complex behavior in cases where one appears to
# be merging topics (E.g. there are existing messages in the
# target topic).
if moved_all_visible_messages:
assert stream_being_edited is not None
assert topic_name is not None or new_stream is not None
for muting_user in get_users_muting_topic(stream_being_edited.id,
orig_topic_name):
# TODO: Ideally, this would be a bulk update operation,
# because we are doing database operations in a loop here.
#
# This loop is only acceptable in production because it is
# rare for more than a few users to have muted an
# individual topic that is being moved; as of this
# writing, no individual topic in Zulip Cloud had been
# muted by more than 100 users.
if new_stream is not None and muting_user.id in delete_event_notify_user_ids:
# If the messages are being moved to a stream the user
# cannot access, then we treat this as the
# messages/topic being deleted for this user. This is
# important for security reasons; we don't want to
# give users a UserTopic row in a stream they cannot
# access. Unmute the topic for such users.
do_unmute_topic(muting_user, stream_being_edited,
orig_topic_name)
else:
# Otherwise, we move the muted topic record for the user.
# We call remove_topic_mute rather than do_unmute_topic to
# avoid sending two events with new muted topics in
# immediate succession; this is correct only because
# muted_topics events always send the full set of topics.
remove_topic_mute(muting_user, stream_being_edited.id,
orig_topic_name)
do_mute_topic(
muting_user,
new_stream
if new_stream is not None else stream_being_edited,
topic_name if topic_name is not None else orig_topic_name,
ignore_duplicate=True,
)
send_event(user_profile.realm, event, users_to_be_notified)
if len(
changed_messages
) > 0 and new_stream is not None and stream_being_edited is not None:
# Notify users that the topic was moved.
changed_messages_count = len(changed_messages)
old_thread_notification_string = None
if send_notification_to_old_thread:
if moved_all_visible_messages:
old_thread_notification_string = gettext_lazy(
"This topic was moved to {new_location} by {user}.")
elif changed_messages_count == 1:
old_thread_notification_string = gettext_lazy(
"A message was moved from this topic to {new_location} by {user}."
)
else:
old_thread_notification_string = gettext_lazy(
"{changed_messages_count} messages were moved from this topic to {new_location} by {user}."
)
new_thread_notification_string = None
if send_notification_to_new_thread:
if moved_all_visible_messages:
new_thread_notification_string = gettext_lazy(
"This topic was moved here from {old_location} by {user}.")
elif changed_messages_count == 1:
new_thread_notification_string = gettext_lazy(
"A message was moved here from {old_location} by {user}.")
else:
new_thread_notification_string = gettext_lazy(
"{changed_messages_count} messages were moved here from {old_location} by {user}."
)
send_message_moved_breadcrumbs(
user_profile,
stream_being_edited,
orig_topic_name,
old_thread_notification_string,
new_stream,
topic_name,
new_thread_notification_string,
changed_messages_count,
)
if (topic_name is not None and new_stream is None and content is None
and len(changed_messages) > 0):
assert stream_being_edited is not None
maybe_send_resolve_topic_notifications(
user_profile=user_profile,
stream=stream_being_edited,
old_topic=orig_topic_name,
new_topic=topic_name,
changed_messages=changed_messages,
)
return len(changed_messages)
def do_test_topic(self, msg: Message,
expected_topic: Optional[str]) -> None:
if expected_topic is not None:
self.assertEqual(msg.topic_name(), expected_topic)
def handle(self, *args, **kwargs):
get_UserMessages_to_expunge().delete()
Message.remove_unreachable()
