def _deleted_message_in_huddle_missed_stream_messages(self, send_as_user: bool,
mock_random_token: MagicMock) -> None:
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
msg_id = self.send_huddle_message(
self.example_email('othello'),
[
self.example_email('hamlet'),
self.example_email('iago'),
],
'Group personal message!',
)
hamlet = self.example_user('hamlet')
iago = self.example_user('iago')
email = self.example_email('othello')
self.login(email)
result = self.client_patch('/json/messages/' + str(msg_id),
{'message_id': msg_id, 'content': ' '})
self.assert_json_success(result)
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id}])
self.assertEqual(len(mail.outbox), 0)
handle_missedmessage_emails(iago.id, [{'message_id': msg_id}])
self.assertEqual(len(mail.outbox), 0)
def _test_cases(self, tokens: List[str], msg_id: int, body: str, email_subject: str,
send_as_user: bool, verify_html_body: bool=False,
show_message_content: bool=True,
verify_body_does_not_include: Optional[List[str]]=None,
trigger: str='') -> None:
othello = self.example_user('othello')
hamlet = self.example_user('hamlet')
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id, 'trigger': trigger}])
if settings.EMAIL_GATEWAY_PATTERN != "":
reply_to_addresses = [settings.EMAIL_GATEWAY_PATTERN % (u'mm' + t) for t in tokens]
reply_to_emails = [formataddr(("Zulip", address)) for address in reply_to_addresses]
else:
reply_to_emails = ["noreply@testserver"]
msg = mail.outbox[0]
from_email = formataddr(("Zulip missed messages", FromAddress.NOREPLY))
self.assertEqual(len(mail.outbox), 1)
if send_as_user:
from_email = '"%s" <%s>' % (othello.full_name, othello.email)
self.assertEqual(msg.from_email, from_email)
self.assertEqual(msg.subject, email_subject)
self.assertEqual(len(msg.reply_to), 1)
self.assertIn(msg.reply_to[0], reply_to_emails)
if verify_html_body:
self.assertIn(body, self.normalize_string(msg.alternatives[0][0]))
else:
self.assertIn(body, self.normalize_string(msg.body))
if verify_body_does_not_include is not None:
for text in verify_body_does_not_include:
self.assertNotIn(text, self.normalize_string(msg.body))
def test_multiple_stream_messages_different_topics(
self, mock_random_token: MagicMock) -> None:
"""Should receive separate emails for each topic within a stream."""
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_email('othello'),
"Denmark", 'Message1')
msg_id_2 = self.send_stream_message(self.example_email('iago'),
"Denmark",
'Message2',
topic_name="test2")
handle_missedmessage_emails(hamlet.id, [
{
'message_id': msg_id_1,
"trigger": "stream_email_notify"
},
{
'message_id': msg_id_2,
"trigger": "stream_email_notify"
},
])
self.assertEqual(len(mail.outbox), 2)
email_subjects = {mail.outbox[0].subject, mail.outbox[1].subject}
valid_email_subjects = {
'New messages in Denmark > test', 'New messages in Denmark > test2'
}
self.assertEqual(email_subjects, valid_email_subjects)
def _test_cases(self, tokens: List[str], msg_id: int, body: str, subject: str,
send_as_user: bool, verify_html_body: bool=False,
show_message_content: bool=True,
verify_body_does_not_include: Optional[List[str]]=None) -> None:
othello = self.example_user('othello')
hamlet = self.example_user('hamlet')
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id}])
if settings.EMAIL_GATEWAY_PATTERN != "":
reply_to_addresses = [settings.EMAIL_GATEWAY_PATTERN % (u'mm' + t) for t in tokens]
reply_to_emails = [formataddr(("Zulip", address)) for address in reply_to_addresses]
else:
reply_to_emails = ["noreply@testserver"]
msg = mail.outbox[0]
from_email = formataddr(("Zulip missed messages", FromAddress.NOREPLY))
self.assertEqual(len(mail.outbox), 1)
if send_as_user:
from_email = '"%s" <%s>' % (othello.full_name, othello.email)
self.assertEqual(msg.from_email, from_email)
self.assertEqual(msg.subject, subject)
self.assertEqual(len(msg.reply_to), 1)
self.assertIn(msg.reply_to[0], reply_to_emails)
if verify_html_body:
self.assertIn(body, self.normalize_string(msg.alternatives[0][0]))
else:
self.assertIn(body, self.normalize_string(msg.body))
if verify_body_does_not_include is not None:
for text in verify_body_does_not_include:
self.assertNotIn(text, self.normalize_string(msg.body))
def test_stream_mentions_multiple_people(
self, mock_random_token: MagicMock) -> None:
"""A mention should take precedence over regular stream messages for email subjects.
Each sender who has mentioned a user should appear in the email subject line.
"""
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_email('iago'),
"Denmark", '@**King Hamlet**')
msg_id_2 = self.send_stream_message(self.example_email('othello'),
"Denmark", '@**King Hamlet**')
msg_id_3 = self.send_stream_message(self.example_email('cordelia'),
"Denmark", 'Regular message')
handle_missedmessage_emails(hamlet.id, [
{
'message_id': msg_id_1,
"trigger": "mentioned"
},
{
'message_id': msg_id_2,
"trigger": "mentioned"
},
{
'message_id': msg_id_3,
"trigger": "stream_email_notify"
},
])
self.assertEqual(len(mail.outbox), 1)
email_subject = 'Iago, Othello, the Moor of Venice mentioned you'
self.assertEqual(mail.outbox[0].subject, email_subject)
def _deleted_message_in_huddle_missed_stream_messages(
self, send_as_user: bool, mock_random_token: MagicMock) -> None:
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
msg_id = self.send_huddle_message(
self.example_email('othello'),
[
self.example_email('hamlet'),
self.example_email('iago'),
],
'Group personal message!',
)
hamlet = self.example_user('hamlet')
iago = self.example_user('iago')
email = self.example_email('othello')
self.login(email)
result = self.client_patch('/json/messages/' + str(msg_id), {
'message_id': msg_id,
'content': ' '
})
self.assert_json_success(result)
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id}])
self.assertEqual(len(mail.outbox), 0)
handle_missedmessage_emails(iago.id, [{'message_id': msg_id}])
self.assertEqual(len(mail.outbox), 0)
def test_multiple_missed_personal_messages(
self, mock_random_token: MagicMock) -> None:
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_personal_message(self.example_email('othello'),
hamlet.email,
'Personal Message 1')
msg_id_2 = self.send_personal_message(self.example_email('iago'),
hamlet.email,
'Personal Message 2')
handle_missedmessage_emails(hamlet.id, [
{
'message_id': msg_id_1
},
{
'message_id': msg_id_2
},
])
self.assertEqual(len(mail.outbox), 2)
email_subject = 'Othello, the Moor of Venice sent you a message'
self.assertEqual(mail.outbox[0].subject, email_subject)
email_subject = 'Iago sent you a message'
self.assertEqual(mail.outbox[1].subject, email_subject)
def test_stream_mentions_multiple_people(self, mock_random_token: MagicMock) -> None:
"""A mention should take precedence over regular stream messages for email subjects.
Each sender who has mentioned a user should appear in the email subject line.
"""
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_email('iago'),
"Denmark",
'@**King Hamlet**')
msg_id_2 = self.send_stream_message(self.example_email('othello'),
"Denmark",
'@**King Hamlet**')
msg_id_3 = self.send_stream_message(self.example_email('cordelia'),
"Denmark",
'Regular message')
handle_missedmessage_emails(hamlet.id, [
{'message_id': msg_id_1, "trigger": "mentioned"},
{'message_id': msg_id_2, "trigger": "mentioned"},
{'message_id': msg_id_3, "trigger": "stream_email_notify"},
])
self.assertEqual(len(mail.outbox), 1)
email_subject = 'Iago, Othello, the Moor of Venice mentioned you'
self.assertEqual(mail.outbox[0].subject, email_subject)
def _test_cases(self,
tokens,
msg_id,
body,
subject,
send_as_user,
verify_html_body=False):
# type: (List[str], int, str, str, bool, bool) -> None
othello = self.example_user('othello')
hamlet = self.example_user('hamlet')
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id}])
if settings.EMAIL_GATEWAY_PATTERN != "":
reply_to_addresses = [
settings.EMAIL_GATEWAY_PATTERN % (u'mm' + t) for t in tokens
]
reply_to_emails = [
formataddr(("Zulip", address))
for address in reply_to_addresses
]
else:
reply_to_emails = ["noreply@testserver"]
msg = mail.outbox[0]
from_email = formataddr(("Zulip missed messages", FromAddress.NOREPLY))
self.assertEqual(len(mail.outbox), 1)
if send_as_user:
from_email = '"%s" <%s>' % (othello.full_name, othello.email)
self.assertEqual(msg.from_email, from_email)
self.assertEqual(msg.subject, subject)
self.assertEqual(len(msg.reply_to), 1)
self.assertIn(msg.reply_to[0], reply_to_emails)
if verify_html_body:
self.assertIn(body, self.normalize_string(msg.alternatives[0][0]))
else:
self.assertIn(body, self.normalize_string(msg.body))
def consume_batch(self, missed_events: List[Dict[str, Any]]) -> None:
by_recipient = defaultdict(list) # type: Dict[int, List[Dict[str, Any]]]
for event in missed_events:
logging.debug("Received missedmessage_emails event: %s" % (event,))
by_recipient[event['user_profile_id']].append(event)
for user_profile_id, events in by_recipient.items():
handle_missedmessage_emails(user_profile_id, events)
def consume_batch(self, missed_events: List[Dict[str, Any]]) -> None:
by_recipient = defaultdict(list) # type: Dict[int, List[Dict[str, Any]]]
for event in missed_events:
logging.debug("Received missedmessage_emails event: %s" % (event,))
by_recipient[event['user_profile_id']].append(event)
for user_profile_id, events in by_recipient.items():
handle_missedmessage_emails(user_profile_id, events)
def start(self):
while True:
missed_events = self.q.drain_queue("missedmessage_emails", json=True)
by_recipient = defaultdict(list) # type: Dict[int, List[Dict[str, Any]]]
for event in missed_events:
logging.info("Received event: %s" % (event,))
by_recipient[event['user_profile_id']].append(event)
for user_profile_id, events in by_recipient.items():
handle_missedmessage_emails(user_profile_id, events)
reset_queries()
# Aggregate all messages received every 2 minutes to let someone finish sending a batch
# of messages
time.sleep(2 * 60)
def start(self):
while True:
missed_events = self.q.drain_queue("missedmessage_emails", json=True)
by_recipient = defaultdict(list) # type: Dict[int, List[Dict[str, Any]]]
for event in missed_events:
logging.info("Received event: %s" % (event,))
by_recipient[event['user_profile_id']].append(event)
for user_profile_id, events in by_recipient.items():
handle_missedmessage_emails(user_profile_id, events)
reset_queries()
# Aggregate all messages received every 2 minutes to let someone finish sending a batch
# of messages
time.sleep(2 * 60)
def _deleted_message_in_missed_stream_messages(self, send_as_user: bool,
mock_random_token: MagicMock) -> None:
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
msg_id = self.send_stream_message(
self.example_email('othello'), "denmark",
'@**King Hamlet** to be deleted')
hamlet = self.example_user('hamlet')
email = self.example_email('othello')
self.login(email)
result = self.client_patch('/json/messages/' + str(msg_id),
{'message_id': msg_id, 'content': ' '})
self.assert_json_success(result)
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id}])
self.assertEqual(len(mail.outbox), 0)
def _deleted_message_in_missed_stream_messages(self, send_as_user: bool,
mock_random_token: MagicMock) -> None:
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
msg_id = self.send_stream_message(
self.example_email('othello'), "denmark",
'@**King Hamlet** to be deleted')
hamlet = self.example_user('hamlet')
email = self.example_email('othello')
self.login(email)
result = self.client_patch('/json/messages/' + str(msg_id),
{'message_id': msg_id, 'content': ' '})
self.assert_json_success(result)
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id}])
self.assertEqual(len(mail.outbox), 0)
def _deleted_message_in_personal_missed_stream_messages(self, send_as_user, mock_random_token):
# type: (bool, MagicMock) -> None
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
msg_id = self.send_personal_message(self.example_email('othello'),
self.example_email('hamlet'),
'Extremely personal message! to be deleted!')
hamlet = self.example_user('hamlet')
email = self.example_email('othello')
self.login(email)
result = self.client_patch('/json/messages/' + str(msg_id),
{'message_id': msg_id, 'content': ' '})
self.assert_json_success(result)
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id}])
self.assertEqual(len(mail.outbox), 0)
def maybe_send_batched_emails(self) -> None:
self.stop_timer()
current_time = time.time()
for user_profile_id, timestamp in list(self.batch_start_by_recipient.items()):
if current_time - timestamp < self.BATCH_DURATION:
continue
events = self.events_by_recipient[user_profile_id]
logging.info("Batch-processing %s missedmessage_emails events for user %s" %
(len(events), user_profile_id))
handle_missedmessage_emails(user_profile_id, events)
del self.events_by_recipient[user_profile_id]
del self.batch_start_by_recipient[user_profile_id]
# By only restarting the timer if there are actually events in
# the queue, we ensure this queue processor is idle when there
# are no missed-message emails to process.
if len(self.batch_start_by_recipient) > 0:
self.ensure_timer()
def maybe_send_batched_emails(self) -> None:
self.stop_timer()
current_time = time.time()
for user_profile_id, timestamp in list(self.batch_start_by_recipient.items()):
if current_time - timestamp < self.BATCH_DURATION:
continue
events = self.events_by_recipient[user_profile_id]
logging.info("Batch-processing %s missedmessage_emails events for user %s" %
(len(events), user_profile_id))
handle_missedmessage_emails(user_profile_id, events)
del self.events_by_recipient[user_profile_id]
del self.batch_start_by_recipient[user_profile_id]
# By only restarting the timer if there are actually events in
# the queue, we ensure this queue processor is idle when there
# are no missed-message emails to process.
if len(self.batch_start_by_recipient) > 0:
self.ensure_timer()
def test_multiple_stream_messages(self, mock_random_token: MagicMock) -> None:
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_email('othello'),
"Denmark",
'Message1')
msg_id_2 = self.send_stream_message(self.example_email('iago'),
"Denmark",
'Message2')
handle_missedmessage_emails(hamlet.id, [
{'message_id': msg_id_1, "trigger": "stream_email_notify"},
{'message_id': msg_id_2, "trigger": "stream_email_notify"},
])
self.assertEqual(len(mail.outbox), 1)
email_subject = 'New messages in Denmark > test'
self.assertEqual(mail.outbox[0].subject, email_subject)
def test_multiple_stream_messages(self, mock_random_token: MagicMock) -> None:
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_email('othello'),
"Denmark",
'Message1')
msg_id_2 = self.send_stream_message(self.example_email('iago'),
"Denmark",
'Message2')
handle_missedmessage_emails(hamlet.id, [
{'message_id': msg_id_1, "trigger": "stream_email_notify"},
{'message_id': msg_id_2, "trigger": "stream_email_notify"},
])
self.assertEqual(len(mail.outbox), 1)
email_subject = 'New messages in Denmark > test'
self.assertEqual(mail.outbox[0].subject, email_subject)
def _deleted_message_in_personal_missed_stream_messages(
self, send_as_user, mock_random_token):
# type: (bool, MagicMock) -> None
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
msg_id = self.send_message(
self.example_email('othello'), self.example_email('hamlet'),
Recipient.PERSONAL, 'Extremely personal message! to be deleted!')
hamlet = self.example_user('hamlet')
email = self.example_email('othello')
self.login(email)
result = self.client_patch('/json/messages/' + str(msg_id), {
'message_id': msg_id,
'content': ' '
})
self.assert_json_success(result)
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id}])
self.assertEqual(len(mail.outbox), 0)
def test_multiple_missed_personal_messages(self, mock_random_token: MagicMock) -> None:
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_personal_message(self.example_email('othello'),
hamlet.email,
'Personal Message 1')
msg_id_2 = self.send_personal_message(self.example_email('iago'),
hamlet.email,
'Personal Message 2')
handle_missedmessage_emails(hamlet.id, [
{'message_id': msg_id_1},
{'message_id': msg_id_2},
])
self.assertEqual(len(mail.outbox), 2)
email_subject = 'Othello, the Moor of Venice sent you a message'
self.assertEqual(mail.outbox[0].subject, email_subject)
email_subject = 'Iago sent you a message'
self.assertEqual(mail.outbox[1].subject, email_subject)
def test_multiple_stream_messages_different_topics(self, mock_random_token: MagicMock) -> None:
"""Should receive separate emails for each topic within a stream."""
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
hamlet = self.example_user('hamlet')
msg_id_1 = self.send_stream_message(self.example_email('othello'),
"Denmark",
'Message1')
msg_id_2 = self.send_stream_message(self.example_email('iago'),
"Denmark",
'Message2',
topic_name="test2")
handle_missedmessage_emails(hamlet.id, [
{'message_id': msg_id_1, "trigger": "stream_email_notify"},
{'message_id': msg_id_2, "trigger": "stream_email_notify"},
])
self.assertEqual(len(mail.outbox), 2)
email_subjects = {mail.outbox[0].subject, mail.outbox[1].subject}
valid_email_subjects = {'New messages in Denmark > test', 'New messages in Denmark > test2'}
self.assertEqual(email_subjects, valid_email_subjects)
def test_message_access_in_emails(self,
mock_random_token: MagicMock) -> None:
# Messages sent to a protected history-private stream shouldn't be
# accessible/available in emails before subscribing
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
stream_name = "private_stream"
self.make_stream(stream_name,
invite_only=True,
history_public_to_subscribers=False)
user = self.example_user('iago')
self.subscribe(user, stream_name)
late_subscribed_user = self.example_user('hamlet')
self.send_stream_message(user.email, stream_name, 'Before subscribing')
self.subscribe(late_subscribed_user, stream_name)
self.send_stream_message(user.email, stream_name, "After subscribing")
mention_msg_id = self.send_stream_message(user.email, stream_name,
'@**King Hamlet**')
handle_missedmessage_emails(late_subscribed_user.id, [
{
'message_id': mention_msg_id,
"trigger": "mentioned"
},
])
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject,
'Iago mentioned you') # email subject
email_text = mail.outbox[0].message().as_string()
self.assertNotIn('Before subscribing', email_text)
self.assertIn('After subscribing', email_text)
self.assertIn('@**King Hamlet**', email_text)
def _test_cases(self, tokens, msg_id, body, subject, send_as_user, verify_html_body=False):
# type: (List[str], int, str, str, bool, bool) -> None
othello = self.example_user('othello')
hamlet = self.example_user('hamlet')
handle_missedmessage_emails(hamlet.id, [{'message_id': msg_id}])
if settings.EMAIL_GATEWAY_PATTERN != "":
reply_to_addresses = [settings.EMAIL_GATEWAY_PATTERN % (u'mm' + t) for t in tokens]
reply_to_emails = [formataddr(("Zulip", address)) for address in reply_to_addresses]
else:
reply_to_emails = ["noreply@testserver"]
msg = mail.outbox[0]
from_email = formataddr(("Zulip missed messages", FromAddress.NOREPLY))
self.assertEqual(len(mail.outbox), 1)
if send_as_user:
from_email = '"%s" <%s>' % (othello.full_name, othello.email)
self.assertEqual(msg.from_email, from_email)
self.assertEqual(msg.subject, subject)
self.assertEqual(len(msg.reply_to), 1)
self.assertIn(msg.reply_to[0], reply_to_emails)
if verify_html_body:
self.assertIn(body, self.normalize_string(msg.alternatives[0][0]))
else:
self.assertIn(body, self.normalize_string(msg.body))
def test_message_access_in_emails(self, mock_random_token: MagicMock) -> None:
# Messages sent to a protected history-private stream shouldn't be
# accessible/available in emails before subscribing
tokens = self._get_tokens()
mock_random_token.side_effect = tokens
stream_name = "private_stream"
self.make_stream(stream_name, invite_only=True,
history_public_to_subscribers=False)
user = self.example_user('iago')
self.subscribe(user, stream_name)
late_subscribed_user = self.example_user('hamlet')
self.send_stream_message(user.email,
stream_name,
'Before subscribing')
self.subscribe(late_subscribed_user, stream_name)
self.send_stream_message(user.email,
stream_name,
"After subscribing")
mention_msg_id = self.send_stream_message(user.email,
stream_name,
'@**King Hamlet**')
handle_missedmessage_emails(late_subscribed_user.id, [
{'message_id': mention_msg_id, "trigger": "mentioned"},
])
self.assertEqual(len(mail.outbox), 1)
self.assertEqual(mail.outbox[0].subject, 'Iago mentioned you') # email subject
email_text = mail.outbox[0].message().as_string()
self.assertNotIn('Before subscribing', email_text)
self.assertIn('After subscribing', email_text)
self.assertIn('@**King Hamlet**', email_text)
