def test_multiple_stream_messages_different_topics(
self, mock_random_token: MagicMock) -> None:
"""Should receive separate emails for each topic within a stream."""
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_email('othello'),
"Denmark", 'Message1')
msg_id_2 = self.send_stream_message(self.example_email('iago'),
"Denmark",
'Message2',
topic_name="test2")
handle_missedmessage_emails(hamlet.id, [
{
'message_id': msg_id_1,
"trigger": "stream_email_notify"
},
{
'message_id': msg_id_2,
"trigger": "stream_email_notify"
},
])
self.assertEqual(len(mail.outbox), 2)
email_subjects = {mail.outbox[0].subject, mail.outbox[1].subject}
valid_email_subjects = {'#Denmark > test', '#Denmark > test2'}
self.assertEqual(email_subjects, valid_email_subjects)
def maybe_send_batched_emails(self) -> None:
with self.lock:
# self.timer_event just triggered execution of this
# function in a thread, so now that we hold the lock, we
# clear the timer_event attribute to record that no Timer
# is active.
self.timer_event = None
current_time = time.time()
for user_profile_id, timestamp in list(self.batch_start_by_recipient.items()):
if current_time - timestamp < self.BATCH_DURATION:
continue
events = self.events_by_recipient[user_profile_id]
logging.info("Batch-processing %s missedmessage_emails events for user %s",
len(events), user_profile_id)
handle_missedmessage_emails(user_profile_id, events)
del self.events_by_recipient[user_profile_id]
del self.batch_start_by_recipient[user_profile_id]
# By only restarting the timer if there are actually events in
# the queue, we ensure this queue processor is idle when there
# are no missed-message emails to process. This avoids
# constant CPU usage when there is no work to do.
if len(self.batch_start_by_recipient) > 0:
self.ensure_timer()
def test_message_access_in_emails(self) -> None:
# Messages sent to a protected history-private stream shouldn't be
# accessible/available in emails before subscribing
stream_name = "private_stream"
self.make_stream(stream_name, invite_only=True,
history_public_to_subscribers=False)
user = self.example_user('iago')
self.subscribe(user, stream_name)
late_subscribed_user = self.example_user('hamlet')
self.send_stream_message(user.email,
stream_name,
'Before subscribing')
self.subscribe(late_subscribed_user, stream_name)
self.send_stream_message(user.email,
stream_name,
"After subscribing")
mention_msg_id = self.send_stream_message(user.email,
stream_name,
'@**King Hamlet**')
handle_missedmessage_emails(late_subscribed_user.id, [
{'message_id': mention_msg_id, "trigger": "mentioned"},
])
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, '#private_stream > test') # email subject
email_text = mail.outbox[0].message().as_string()
self.assertNotIn('Before subscribing', email_text)
self.assertIn('After subscribing', email_text)
self.assertIn('@**King Hamlet**', email_text)
def test_multiple_stream_messages_and_mentions(
self, mock_random_token: MagicMock) -> None:
"""Subject should be stream name and topic as usual."""
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_email('iago'),
"Denmark", 'Regular message')
msg_id_2 = self.send_stream_message(self.example_email('othello'),
"Denmark", '@**King Hamlet**')
handle_missedmessage_emails(hamlet.id, [
{
'message_id': msg_id_1,
"trigger": "stream_email_notify"
},
{
'message_id': msg_id_2,
"trigger": "mentioned"
},
])
self.assertEqual(len(mail.outbox), 1)
email_subject = '#Denmark > test'
self.assertEqual(mail.outbox[0].subject, email_subject)
def test_sender_name_in_missed_message(self) -> None:
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_email('iago'),
"Denmark",
'@**King Hamlet**')
msg_id_2 = self.send_stream_message(self.example_email('iago'),
"Verona",
'* 1\n *2')
msg_id_3 = self.send_personal_message(self.example_email('iago'),
hamlet.email,
'Hello')
handle_missedmessage_emails(hamlet.id, [
{'message_id': msg_id_1, "trigger": "mentioned"},
{'message_id': msg_id_2, "trigger": "stream_email_notify"},
{'message_id': msg_id_3},
])
self.assertIn('Iago: @**King Hamlet**\n\n--\nYou are', mail.outbox[0].body)
# If message content starts with <p> tag the sender name is appended inside the <p> tag.
self.assertIn('<p><b>Iago</b>: <span class="user-mention"', mail.outbox[0].alternatives[0][0])
self.assertIn('Iago: * 1\n *2\n\n--\nYou are receiving', mail.outbox[1].body)
# If message content does not starts with <p> tag sender name is appended before the <p> tag
self.assertIn(' <b>Iago</b>: <ul>\n<li>1<br/>\n *2</li>\n</ul>\n',
mail.outbox[1].alternatives[0][0])
self.assertEqual('Hello\n\n--\n\nReply', mail.outbox[2].body[:16])
# Sender name is not appended to message for PM missed messages
self.assertIn('>\n \n <p>Hello</p>\n',
mail.outbox[2].alternatives[0][0])
def _test_cases(self, msg_id: int, body: str, email_subject: str,
send_as_user: bool, verify_html_body: bool=False,
show_message_content: bool=True,
verify_body_does_not_include: Optional[List[str]]=None,
trigger: str='') -> None:
othello = self.example_user('othello')
hamlet = self.example_user('hamlet')
tokens = self._get_tokens()
with patch('zerver.lib.email_mirror.generate_missed_message_token', side_effect=tokens):
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id, 'trigger': trigger}])
if settings.EMAIL_GATEWAY_PATTERN != "":
reply_to_addresses = [settings.EMAIL_GATEWAY_PATTERN % (t,) for t in tokens]
reply_to_emails = [formataddr(("Zulip", address)) for address in reply_to_addresses]
else:
reply_to_emails = ["noreply@testserver"]
msg = mail.outbox[0]
from_email = formataddr(("Zulip missed messages", FromAddress.NOREPLY))
self.assertEqual(len(mail.outbox), 1)
if send_as_user:
from_email = '"%s" <%s>' % (othello.full_name, othello.email)
self.assertEqual(msg.from_email, from_email)
self.assertEqual(msg.subject, email_subject)
self.assertEqual(len(msg.reply_to), 1)
self.assertIn(msg.reply_to[0], reply_to_emails)
if verify_html_body:
self.assertIn(body, self.normalize_string(msg.alternatives[0][0]))
else:
self.assertIn(body, self.normalize_string(msg.body))
if verify_body_does_not_include is not None:
for text in verify_body_does_not_include:
self.assertNotIn(text, self.normalize_string(msg.body))
def test_multiple_missed_personal_messages(
self, mock_random_token: MagicMock) -> None:
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_personal_message(self.example_email('othello'),
hamlet.email,
'Personal Message 1')
msg_id_2 = self.send_personal_message(self.example_email('iago'),
hamlet.email,
'Personal Message 2')
handle_missedmessage_emails(hamlet.id, [
{
'message_id': msg_id_1
},
{
'message_id': msg_id_2
},
])
self.assertEqual(len(mail.outbox), 2)
email_subject = 'PMs with Othello, the Moor of Venice'
self.assertEqual(mail.outbox[0].subject, email_subject)
email_subject = 'PMs with Iago'
self.assertEqual(mail.outbox[1].subject, email_subject)
def test_stream_mentions_multiple_people(self, mock_random_token: MagicMock) -> None:
"""A mention should take precedence over regular stream messages for email subjects.
Each sender who has mentioned a user should appear in the email subject line.
"""
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_email('iago'),
"Denmark",
'@**King Hamlet**')
msg_id_2 = self.send_stream_message(self.example_email('othello'),
"Denmark",
'@**King Hamlet**')
msg_id_3 = self.send_stream_message(self.example_email('cordelia'),
"Denmark",
'Regular message')
handle_missedmessage_emails(hamlet.id, [
{'message_id': msg_id_1, "trigger": "mentioned"},
{'message_id': msg_id_2, "trigger": "mentioned"},
{'message_id': msg_id_3, "trigger": "stream_email_notify"},
])
self.assertEqual(len(mail.outbox), 1)
email_subject = 'Iago, Othello, the Moor of Venice mentioned you'
self.assertEqual(mail.outbox[0].subject, email_subject)
def _deleted_message_in_huddle_missed_stream_messages(self, send_as_user: bool,
mock_random_token: MagicMock) -> None:
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
msg_id = self.send_huddle_message(
self.example_email('othello'),
[
self.example_email('hamlet'),
self.example_email('iago'),
],
'Group personal message!',
)
hamlet = self.example_user('hamlet')
iago = self.example_user('iago')
email = self.example_email('othello')
self.login(email)
result = self.client_patch('/json/messages/' + str(msg_id),
{'message_id': msg_id, 'content': ' '})
self.assert_json_success(result)
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id}])
self.assertEqual(len(mail.outbox), 0)
handle_missedmessage_emails(iago.id, [{'message_id': msg_id}])
self.assertEqual(len(mail.outbox), 0)
def maybe_send_batched_emails(self) -> None:
with self.lock:
# self.timer_event just triggered execution of this
# function in a thread, so now that we hold the lock, we
# clear the timer_event attribute to record that no Timer
# is active.
self.timer_event = None
current_time = timezone_now()
with transaction.atomic():
events_to_process = ScheduledMessageNotificationEmail.objects.filter(
scheduled_timestamp__lte=current_time
).select_related()
# Batch the entries by user
events_by_recipient: Dict[int, List[Dict[str, Any]]] = {}
for event in events_to_process:
entry = dict(
user_profile_id=event.user_profile_id,
message_id=event.message_id,
trigger=event.trigger,
mentioned_user_group_id=event.mentioned_user_group_id,
)
if event.user_profile_id in events_by_recipient:
events_by_recipient[event.user_profile_id].append(entry)
else:
events_by_recipient[event.user_profile_id] = [entry]
for user_profile_id in events_by_recipient.keys():
events: List[Dict[str, Any]] = events_by_recipient[user_profile_id]
logging.info(
"Batch-processing %s missedmessage_emails events for user %s",
len(events),
user_profile_id,
)
try:
# Because we process events in batches, an
# escaped exception here would lead to
# duplicate messages being sent for other
# users in the same events_to_process batch,
# and no guarantee of forward progress.
handle_missedmessage_emails(user_profile_id, events)
except Exception:
logging.exception(
"Failed to process %d missedmessage_emails for user %s",
len(events),
user_profile_id,
stack_info=True,
)
events_to_process.delete()
# By only restarting the timer if there are actually events in
# the queue, we ensure this queue processor is idle when there
# are no missed-message emails to process. This avoids
# constant CPU usage when there is no work to do.
if ScheduledMessageNotificationEmail.objects.exists():
self.ensure_timer()
def test_stream_mentions_multiple_people(self) -> None:
"""Subject should be stream name and topic as usual."""
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_user('iago'),
"Denmark", '@**King Hamlet**')
msg_id_2 = self.send_stream_message(self.example_user('othello'),
"Denmark", '@**King Hamlet**')
msg_id_3 = self.send_stream_message(self.example_user('cordelia'),
"Denmark", 'Regular message')
handle_missedmessage_emails(hamlet.id, [
{
'message_id': msg_id_1,
"trigger": "mentioned"
},
{
'message_id': msg_id_2,
"trigger": "mentioned"
},
{
'message_id': msg_id_3,
"trigger": "stream_email_notify"
},
])
self.assertEqual(len(mail.outbox), 1)
email_subject = '#Denmark > test'
self.assertEqual(mail.outbox[0].subject, email_subject)
def _deleted_message_in_personal_missed_stream_messages(self, send_as_user: bool) -> None:
msg_id = self.send_personal_message(self.example_user('othello'),
self.example_user('hamlet'),
'Extremely personal message! to be deleted!')
hamlet = self.example_user('hamlet')
self.login('othello')
result = self.client_patch('/json/messages/' + str(msg_id),
{'message_id': msg_id, 'content': ' '})
self.assert_json_success(result)
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id}])
self.assertEqual(len(mail.outbox), 0)
def _deleted_message_in_missed_stream_messages(self, send_as_user: bool) -> None:
msg_id = self.send_stream_message(
self.example_email('othello'), "denmark",
'@**King Hamlet** to be deleted')
hamlet = self.example_user('hamlet')
email = self.example_email('othello')
self.login(email)
result = self.client_patch('/json/messages/' + str(msg_id),
{'message_id': msg_id, 'content': ' '})
self.assert_json_success(result)
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id}])
self.assertEqual(len(mail.outbox), 0)
def test_multiple_stream_messages(self) -> None:
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_email('othello'),
"Denmark",
'Message1')
msg_id_2 = self.send_stream_message(self.example_email('iago'),
"Denmark",
'Message2')
handle_missedmessage_emails(hamlet.id, [
{'message_id': msg_id_1, "trigger": "stream_email_notify"},
{'message_id': msg_id_2, "trigger": "stream_email_notify"},
])
self.assertEqual(len(mail.outbox), 1)
email_subject = '#Denmark > test'
self.assertEqual(mail.outbox[0].subject, email_subject)
def test_multiple_missed_personal_messages(self) -> None:
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_personal_message(self.example_user('othello'),
hamlet,
'Personal Message 1')
msg_id_2 = self.send_personal_message(self.example_user('iago'),
hamlet,
'Personal Message 2')
handle_missedmessage_emails(hamlet.id, [
{'message_id': msg_id_1},
{'message_id': msg_id_2},
])
self.assertEqual(len(mail.outbox), 2)
email_subject = 'PMs with Othello, the Moor of Venice'
self.assertEqual(mail.outbox[0].subject, email_subject)
email_subject = 'PMs with Iago'
self.assertEqual(mail.outbox[1].subject, email_subject)
def test_multiple_stream_messages(self, mock_random_token: MagicMock) -> None:
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_email('othello'),
"Denmark",
'Message1')
msg_id_2 = self.send_stream_message(self.example_email('iago'),
"Denmark",
'Message2')
handle_missedmessage_emails(hamlet.id, [
{'message_id': msg_id_1, "trigger": "stream_email_notify"},
{'message_id': msg_id_2, "trigger": "stream_email_notify"},
])
self.assertEqual(len(mail.outbox), 1)
email_subject = 'New messages in Denmark > test'
self.assertEqual(mail.outbox[0].subject, email_subject)
def maybe_send_batched_emails(self) -> None:
self.stop_timer()
current_time = time.time()
for user_profile_id, timestamp in list(self.batch_start_by_recipient.items()):
if current_time - timestamp < self.BATCH_DURATION:
continue
events = self.events_by_recipient[user_profile_id]
logging.info("Batch-processing %s missedmessage_emails events for user %s",
len(events), user_profile_id)
handle_missedmessage_emails(user_profile_id, events)
del self.events_by_recipient[user_profile_id]
del self.batch_start_by_recipient[user_profile_id]
# By only restarting the timer if there are actually events in
# the queue, we ensure this queue processor is idle when there
# are no missed-message emails to process.
if len(self.batch_start_by_recipient) > 0:
self.ensure_timer()
