def _validate_username(username):
"""Validate the username.
Arguments:
username (unicode): The proposed username.
Returns:
None
Raises:
errors.AccountUsernameInvalid
"""
try:
_validate_unicode(username)
_validate_type(username, basestring, accounts.USERNAME_BAD_TYPE_MSG)
_validate_length(
username,
accounts.USERNAME_MIN_LENGTH,
accounts.USERNAME_MAX_LENGTH,
accounts.USERNAME_BAD_LENGTH_MSG
)
with override_language('en'):
# `validate_username` provides a proper localized message, however the API needs only the English
# message by convention.
student_forms.validate_username(username)
except (UnicodeError, errors.AccountDataBadType, errors.AccountDataBadLength) as username_err:
raise errors.AccountUsernameInvalid(text_type(username_err))
except ValidationError as validation_err:
raise errors.AccountUsernameInvalid(validation_err.message)
def render_message_to_string(subject_template, message_template, param_dict, language=None):
"""
Render a mail subject and message templates using the parameters from
param_dict and the given language. If language is None, the platform
default language is used.
Returns two strings that correspond to the rendered, translated email
subject and message.
"""
with override_language(language):
return get_subject_and_message(subject_template, message_template, param_dict)
def render_message_to_string(subject_template, message_template, param_dict, language=None):
"""
Render a mail subject and message templates using the parameters from
param_dict and the given language. If language is None, the platform
default language is used.
Returns two strings that correspond to the rendered, translated email
subject and message.
"""
# edunext | laq, added so that if language is None, the current language(that comes from the email sender) wont be overriden
if language is None:
return get_subject_and_message(subject_template, message_template, param_dict)
else:
with override_language(language):
return get_subject_and_message(subject_template, message_template, param_dict)
def _get_source_address(course_id, course_title):
"""
Calculates an email address to be used as the 'from-address' for sent emails.
Makes a unique from name and address for each course, e.g.
"COURSE_TITLE" Course Staff <*****@*****.**>
"""
course_title_no_quotes = re.sub(r'"', '', course_title)
# For the email address, get the course. Then make sure that it can be used
# in an email address, by substituting a '_' anywhere a non-(ascii, period, or dash)
# character appears.
course_name = re.sub(r"[^\w.-]", '_', course_id.course)
with override_language(settings.LANGUAGE_CODE):
from_addr_format = u'{name} {email}'.format(
# Translators: Bulk email from address e.g. ("Physics 101" Course Staff)
name=_('"{course_title}" Course Staff'),
email=u'<{course_name}-{from_email}>',
)
def format_address(course_title_no_quotes):
"""
Partial function for formatting the from_addr. Since
`course_title_no_quotes` may be truncated to make sure the returned
string has fewer than 320 characters, we define this function to make
it easy to determine quickly what the max length is for
`course_title_no_quotes`.
"""
return from_addr_format.format(
course_title=course_title_no_quotes,
course_name=course_name,
from_email=settings.BULK_EMAIL_DEFAULT_FROM_EMAIL,
)
from_addr = format_address(course_title_no_quotes)
# If it's longer than 320 characters, reformat, but with the course name
# rather than course title. Amazon SES's from address field appears to have a maximum
# length of 320.
if len(from_addr) >= 320:
from_addr = format_address(course_name)
return from_addr
def test_calendar_template_content(self, override, activate_lang, expected_lang):
""" Test content of GoogleCalendarBlock's rendered views """
# pylint: disable=no-value-for-parameter
if override:
with override_language(activate_lang):
_block, student_fragment, studio_fragment = self._render_calendar_block()
else:
_block, student_fragment, studio_fragment = self._render_calendar_block()
src_url = 'https://www.google.com/calendar/embed?mode=Month&src={id}&showCalendars=0&hl={lang}'.format(
id=DEFAULT_CALENDAR_ID,
lang=expected_lang,
)
assert_in('<div class="google-calendar-xblock-wrapper">', student_fragment.content)
assert_in(cgi.escape(src_url), student_fragment.content)
assert_in('Google Calendar', student_fragment.content)
assert_in(STUDIO_EDIT_WRAPPER, studio_fragment.content)
assert_in(VALIDATION_WRAPPER, studio_fragment.content)
assert_in(USER_INPUTS_WRAPPER, studio_fragment.content)
assert_in(BUTTONS_WRAPPER, studio_fragment.content)
def _validate_username(username):
"""Validate the username.
Arguments:
username (unicode): The proposed username.
Returns:
None
Raises:
AccountUsernameInvalid
"""
if not isinstance(username, basestring):
raise AccountUsernameInvalid(u"Username must be a string")
if len(username) < USERNAME_MIN_LENGTH:
raise AccountUsernameInvalid(
u"Username '{username}' must be at least {min} characters long".format(
username=username,
min=USERNAME_MIN_LENGTH
)
)
if len(username) > USERNAME_MAX_LENGTH:
raise AccountUsernameInvalid(
u"Username '{username}' must be at most {max} characters long".format(
username=username,
max=USERNAME_MAX_LENGTH
)
)
try:
with override_language('en'):
# `validate_username` provides a proper localized message, however the API needs only the English
# message by convention.
student_forms.validate_username(username)
except ValidationError as error:
raise AccountUsernameInvalid(error.message)
def test_locale(self):
with override_language('fr'):
self.assertEqual(text_type(StandardScopeClaims.info_profile[0]),
'Profil de base')
def _get_source_address(course_id, course_title, course_language, truncate=True):
"""
Calculates an email address to be used as the 'from-address' for sent emails.
Makes a unique from name and address for each course, e.g.
"COURSE_TITLE" Course Staff <*****@*****.**>
If, when decoded to ascii, this from_addr is longer than 320 characters,
use the course_name rather than the course title, e.g.
"course_name" Course Staff <*****@*****.**>
The "truncate" kwarg is only used for tests.
"""
course_title_no_quotes = re.sub(r'"', '', course_title)
# For the email address, get the course. Then make sure that it can be used
# in an email address, by substituting a '_' anywhere a non-(ascii, period, or dash)
# character appears.
course_name = re.sub(r"[^\w.-]", '_', course_id.course)
# Use course.language if present
language = course_language if course_language else settings.LANGUAGE_CODE
with override_language(language):
# RFC2821 requires the byte order of the email address to be the name then email
# e.g. "John Doe <*****@*****.**>"
# Although the display will be flipped in RTL languages, the byte order is still the same.
from_addr_format = u'{name} {email}'.format(
# Translators: Bulk email from address e.g. ("Physics 101" Course Staff)
name=_('"{course_title}" Course Staff'),
email=u'<{course_name}-{from_email}>',
)
def format_address(course_title_no_quotes):
"""
Partial function for formatting the from_addr. Since
`course_title_no_quotes` may be truncated to make sure the returned
string has fewer than 320 characters, we define this function to make
it easy to determine quickly what the max length is for
`course_title_no_quotes`.
"""
return from_addr_format.format(
course_title=course_title_no_quotes,
course_name=course_name,
from_email=configuration_helpers.get_value(
'email_from_address',
settings.BULK_EMAIL_DEFAULT_FROM_EMAIL
)
)
from_addr = format_address(course_title_no_quotes)
# If the encoded from_addr is longer than 320 characters, reformat,
# but with the course name rather than course title.
# Amazon SES's from address field appears to have a maximum length of 320.
__, encoded_from_addr = forbid_multi_line_headers('from', from_addr, 'utf-8')
# It seems that this value is also escaped when set out to amazon, judging
# from our logs
escaped_encoded_from_addr = escape(encoded_from_addr)
if len(escaped_encoded_from_addr) >= 320 and truncate:
from_addr = format_address(course_name)
return from_addr
def test_platform_language_is_used_for_logged_in_user(self):
with override_language('zh_CN'): # simulate a user login
subject, message = self.get_subject_and_message(None)
self.assertIn("You have been", subject)
self.assertIn("You have been", message)
def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile,
missed_messages: List[Dict[
str, Any]],
message_count: int) -> None:
"""
Send a reminder email to a user if she's missed some PMs by being offline.
The email will have its reply to address set to a limited used email
address that will send a zulip message to the correct recipient. This
allows the user to respond to missed PMs, huddles, and @-mentions directly
from the email.
`user_profile` is the user to send the reminder to
`missed_messages` is a list of dictionaries to Message objects and other data
for a group of messages that share a recipient (and topic)
"""
from zerver.context_processors import common_context
# Disabled missedmessage emails internally
if not user_profile.enable_offline_email_notifications:
return
recipients = {(msg['message'].recipient_id, msg['message'].topic_name())
for msg in missed_messages}
if len(recipients) != 1:
raise ValueError(
'All missed_messages must have the same recipient and topic %r' %
(recipients, ), )
# This link is no longer a part of the email, but keeping the code in case
# we find a clean way to add it back in the future
unsubscribe_link = one_click_unsubscribe_link(user_profile,
"missed_messages")
context = common_context(user_profile)
context.update({
'name':
user_profile.full_name,
'message_count':
message_count,
'unsubscribe_link':
unsubscribe_link,
'realm_name_in_notifications':
user_profile.realm_name_in_notifications,
})
triggers = list(message['trigger'] for message in missed_messages)
unique_triggers = set(triggers)
context.update({
'mention':
'mentioned' in unique_triggers
or 'wildcard_mentioned' in unique_triggers,
'stream_email_notify':
'stream_email_notify' in unique_triggers,
'mention_count':
triggers.count('mentioned') + triggers.count("wildcard_mentioned"),
})
# If this setting (email mirroring integration) is enabled, only then
# can users reply to email to send message to Zulip. Thus, one must
# ensure to display warning in the template.
if settings.EMAIL_GATEWAY_PATTERN:
context.update({
'reply_to_zulip': True,
})
else:
context.update({
'reply_to_zulip': False,
})
from zerver.lib.email_mirror import create_missed_message_address
reply_to_address = create_missed_message_address(
user_profile, missed_messages[0]['message'])
if reply_to_address == FromAddress.NOREPLY:
reply_to_name = None
else:
reply_to_name = "Zulip"
narrow_url = get_narrow_url(user_profile, missed_messages[0]['message'])
context.update({
'narrow_url': narrow_url,
})
senders = list({m['message'].sender for m in missed_messages})
if (missed_messages[0]['message'].recipient.type == Recipient.HUDDLE):
display_recipient = get_display_recipient(
missed_messages[0]['message'].recipient)
# Make sure that this is a list of strings, not a string.
assert not isinstance(display_recipient, str)
other_recipients = [
r['full_name'] for r in display_recipient
if r['id'] != user_profile.id
]
context.update({'group_pm': True})
if len(other_recipients) == 2:
huddle_display_name = " and ".join(other_recipients)
context.update({'huddle_display_name': huddle_display_name})
elif len(other_recipients) == 3:
huddle_display_name = f"{other_recipients[0]}, {other_recipients[1]}, and {other_recipients[2]}"
context.update({'huddle_display_name': huddle_display_name})
else:
huddle_display_name = "{}, and {} others".format(
', '.join(other_recipients[:2]),
len(other_recipients) - 2)
context.update({'huddle_display_name': huddle_display_name})
elif (missed_messages[0]['message'].recipient.type == Recipient.PERSONAL):
context.update({'private_message': True})
elif (context['mention'] or context['stream_email_notify']):
# Keep only the senders who actually mentioned the user
if context['mention']:
senders = list({
m['message'].sender
for m in missed_messages if m['trigger'] == 'mentioned'
or m['trigger'] == 'wildcard_mentioned'
})
message = missed_messages[0]['message']
stream = Stream.objects.only('id',
'name').get(id=message.recipient.type_id)
stream_header = f"{stream.name} > {message.topic_name()}"
context.update({
'stream_header': stream_header,
})
else:
raise AssertionError("Invalid messages!")
# If message content is disabled, then flush all information we pass to email.
if not message_content_allowed_in_missedmessage_emails(user_profile):
realm = user_profile.realm
context.update({
'reply_to_zulip':
False,
'messages': [],
'sender_str':
"",
'realm_str':
realm.name,
'huddle_display_name':
"",
'show_message_content':
False,
'message_content_disabled_by_user':
not user_profile.message_content_in_email_notifications,
'message_content_disabled_by_realm':
not realm.message_content_allowed_in_email_notifications,
})
else:
context.update({
'messages':
build_message_list(user_profile,
list(m['message'] for m in missed_messages)),
'sender_str':
", ".join(sender.full_name for sender in senders),
'realm_str':
user_profile.realm.name,
'show_message_content':
True,
})
with override_language(user_profile.default_language):
from_name: str = _("Zulip missed messages")
from_address = FromAddress.NOREPLY
if len(senders) == 1 and settings.SEND_MISSED_MESSAGE_EMAILS_AS_USER:
# If this setting is enabled, you can reply to the Zulip
# missed message emails directly back to the original sender.
# However, one must ensure the Zulip server is in the SPF
# record for the domain, or there will be spam/deliverability
# problems.
#
# Also, this setting is not really compatible with
# EMAIL_ADDRESS_VISIBILITY_ADMINS.
sender = senders[0]
from_name, from_address = (sender.full_name, sender.email)
context.update({
'reply_to_zulip': False,
})
email_dict = {
'template_prefix': 'zerver/emails/missed_message',
'to_user_ids': [user_profile.id],
'from_name': from_name,
'from_address': from_address,
'reply_to_email': formataddr((reply_to_name, reply_to_address)),
'context': context
}
queue_json_publish("email_senders", email_dict)
user_profile.last_reminder = timezone_now()
user_profile.save(update_fields=['last_reminder'])
def build_email(
template_prefix: str,
to_user_ids: Optional[List[int]] = None,
to_emails: Optional[List[str]] = None,
from_name: Optional[str] = None,
from_address: Optional[str] = None,
reply_to_email: Optional[str] = None,
language: Optional[str] = None,
context: Optional[Dict[str, Any]] = None) -> EmailMultiAlternatives:
# Callers should pass exactly one of to_user_id and to_email.
assert (to_user_ids is None) ^ (to_emails is None)
if to_user_ids is not None:
to_users = [
get_user_profile_by_id(to_user_id) for to_user_id in to_user_ids
]
to_emails = [
formataddr((to_user.full_name, to_user.delivery_email))
for to_user in to_users
]
if context is None:
context = {}
context.update({
'support_email': FromAddress.SUPPORT,
'email_images_base_uri':
settings.ROOT_DOMAIN_URI + '/static/images/emails',
'physical_address': settings.PHYSICAL_ADDRESS,
})
def render_templates() -> Tuple[str, str, str]:
email_subject = loader.render_to_string(
template_prefix + '.subject.txt',
context=context,
using='Jinja2_plaintext').strip().replace('\n', '')
message = loader.render_to_string(template_prefix + '.txt',
context=context,
using='Jinja2_plaintext')
try:
html_message = loader.render_to_string(template_prefix + '.html',
context)
except TemplateDoesNotExist:
emails_dir = os.path.dirname(template_prefix)
template = os.path.basename(template_prefix)
compiled_template_prefix = os.path.join(emails_dir, "compiled",
template)
html_message = loader.render_to_string(
compiled_template_prefix + '.html', context)
return (html_message, message, email_subject)
if not language and to_user_ids is not None:
language = to_users[0].default_language
if language:
with override_language(language):
# Make sure that we render the email using the target's native language
(html_message, message, email_subject) = render_templates()
else:
(html_message, message, email_subject) = render_templates()
logger.warning(
"Missing language for email template '{}'".format(template_prefix))
if from_name is None:
from_name = "Zulip"
if from_address is None:
from_address = FromAddress.NOREPLY
from_email = formataddr((from_name, from_address))
reply_to = None
if reply_to_email is not None:
reply_to = [reply_to_email]
# Remove the from_name in the reply-to for noreply emails, so that users
# see "noreply@..." rather than "Zulip" or whatever the from_name is
# when they reply in their email client.
elif from_address == FromAddress.NOREPLY:
reply_to = [FromAddress.NOREPLY]
mail = EmailMultiAlternatives(email_subject,
message,
from_email,
to_emails,
reply_to=reply_to)
if html_message is not None:
mail.attach_alternative(html_message, 'text/html')
return mail
def build_email(
template_prefix: str,
to_user_ids: Optional[List[int]] = None,
to_emails: Optional[List[str]] = None,
from_name: Optional[str] = None,
from_address: Optional[str] = None,
reply_to_email: Optional[str] = None,
language: Optional[str] = None,
context: Mapping[str, Any] = {},
realm: Optional[Realm] = None,
) -> EmailMultiAlternatives:
# Callers should pass exactly one of to_user_id and to_email.
assert (to_user_ids is None) ^ (to_emails is None)
if to_user_ids is not None:
to_users = [
get_user_profile_by_id(to_user_id) for to_user_id in to_user_ids
]
if realm is None:
assert len({to_user.realm_id for to_user in to_users}) == 1
realm = to_users[0].realm
to_emails = [
str(
Address(display_name=to_user.full_name,
addr_spec=to_user.delivery_email))
for to_user in to_users
]
extra_headers = {}
if realm is not None:
# formaddr is meant for formatting (display_name, email_address) pair for headers like "To",
# but we can use its utility for formatting the List-Id header, as it follows the same format,
# except having just a domain instead of an email address.
extra_headers["List-Id"] = formataddr((realm.name, realm.host))
context = {
**context,
"support_email": FromAddress.SUPPORT,
"email_images_base_uri":
settings.ROOT_DOMAIN_URI + "/static/images/emails",
"physical_address": settings.PHYSICAL_ADDRESS,
}
def render_templates() -> Tuple[str, str, str]:
email_subject = (loader.render_to_string(
template_prefix + ".subject.txt",
context=context,
using="Jinja2_plaintext").strip().replace("\n", ""))
message = loader.render_to_string(template_prefix + ".txt",
context=context,
using="Jinja2_plaintext")
try:
html_message = loader.render_to_string(template_prefix + ".html",
context)
except TemplateDoesNotExist:
emails_dir = os.path.dirname(template_prefix)
template = os.path.basename(template_prefix)
compiled_template_prefix = os.path.join(emails_dir, "compiled",
template)
html_message = loader.render_to_string(
compiled_template_prefix + ".html", context)
return (html_message, message, email_subject)
# The i18n story for emails is a bit complicated. For emails
# going to a single user, we want to use the language that user
# has configured for their Zulip account. For emails going to
# multiple users or to email addresses without a known Zulip
# account (E.g. invitations), we want to use the default language
# configured for the Zulip organization.
#
# See our i18n documentation for some high-level details:
# https://zulip.readthedocs.io/en/latest/translating/internationalization.html
if not language and to_user_ids is not None:
language = to_users[0].default_language
if language:
with override_language(language):
# Make sure that we render the email using the target's native language
(html_message, message, email_subject) = render_templates()
else:
(html_message, message, email_subject) = render_templates()
logger.warning("Missing language for email template '%s'",
template_prefix)
if from_name is None:
from_name = "Zulip"
if from_address is None:
from_address = FromAddress.NOREPLY
if from_address == FromAddress.tokenized_no_reply_placeholder:
from_address = FromAddress.tokenized_no_reply_address()
if from_address == FromAddress.no_reply_placeholder:
from_address = FromAddress.NOREPLY
if from_address == FromAddress.support_placeholder:
from_address = FromAddress.SUPPORT
# Set the "From" that is displayed separately from the envelope-from.
extra_headers["From"] = str(
Address(display_name=from_name, addr_spec=from_address))
# Check ASCII encoding length. Amazon SES rejects emails with
# From names longer than 320 characters (which appears to be a
# misinterpretation of the RFC); in that case we drop the name
# from the From line, under the theory that it's better to send
# the email with a simplified From field than not.
if len(sanitize_address(extra_headers["From"], "utf-8")) > 320:
extra_headers["From"] = str(Address(addr_spec=from_address))
# If we have an unsubscribe link for this email, configure it for
# "Unsubscribe" buttons in email clients via the List-Unsubscribe header.
#
# Note that Microsoft ignores URLs in List-Unsubscribe headers, as
# they only support the alternative `mailto:` format, which we
# have not implemented.
if "unsubscribe_link" in context:
extra_headers["List-Unsubscribe"] = f"<{context['unsubscribe_link']}>"
extra_headers["List-Unsubscribe-Post"] = "List-Unsubscribe=One-Click"
reply_to = None
if reply_to_email is not None:
reply_to = [reply_to_email]
# Remove the from_name in the reply-to for noreply emails, so that users
# see "noreply@..." rather than "Zulip" or whatever the from_name is
# when they reply in their email client.
elif from_address == FromAddress.NOREPLY:
reply_to = [FromAddress.NOREPLY]
envelope_from = FromAddress.NOREPLY
mail = EmailMultiAlternatives(email_subject,
message,
envelope_from,
to_emails,
reply_to=reply_to,
headers=extra_headers)
if html_message is not None:
mail.attach_alternative(html_message, "text/html")
return mail
def add_subscriptions_backend(
request: HttpRequest,
user_profile: UserProfile,
streams_raw: Iterable[Dict[str,
str]] = REQ("subscriptions",
validator=add_subscriptions_schema),
invite_only: bool = REQ(validator=check_bool, default=False),
stream_post_policy: int = REQ(validator=check_int_in(
Stream.STREAM_POST_POLICY_TYPES),
default=Stream.STREAM_POST_POLICY_EVERYONE),
history_public_to_subscribers: Optional[bool] = REQ(validator=check_bool,
default=None),
message_retention_days: Union[str,
int] = REQ(validator=check_string_or_int,
default=RETENTION_DEFAULT),
announce: bool = REQ(validator=check_bool, default=False),
principals: Union[Sequence[str], Sequence[int]] = REQ(
validator=check_principals,
default=EMPTY_PRINCIPALS,
),
authorization_errors_fatal: bool = REQ(validator=check_bool, default=True),
) -> HttpResponse:
stream_dicts = []
color_map = {}
for stream_dict in streams_raw:
# 'color' field is optional
# check for its presence in the streams_raw first
if 'color' in stream_dict:
color_map[stream_dict['name']] = stream_dict['color']
if 'description' in stream_dict:
# We don't allow newline characters in stream descriptions.
stream_dict['description'] = stream_dict['description'].replace(
"\n", " ")
stream_dict_copy: Dict[str, Any] = {}
for field in stream_dict:
stream_dict_copy[field] = stream_dict[field]
# Strip the stream name here.
stream_dict_copy['name'] = stream_dict_copy['name'].strip()
stream_dict_copy["invite_only"] = invite_only
stream_dict_copy["stream_post_policy"] = stream_post_policy
stream_dict_copy[
"history_public_to_subscribers"] = history_public_to_subscribers
stream_dict_copy[
"message_retention_days"] = parse_message_retention_days(
message_retention_days,
Stream.MESSAGE_RETENTION_SPECIAL_VALUES_MAP)
stream_dicts.append(stream_dict_copy)
# Validation of the streams arguments, including enforcement of
# can_create_streams policy and check_stream_name policy is inside
# list_to_streams.
existing_streams, created_streams = \
list_to_streams(stream_dicts, user_profile, autocreate=True)
authorized_streams, unauthorized_streams = \
filter_stream_authorization(user_profile, existing_streams)
if len(unauthorized_streams) > 0 and authorization_errors_fatal:
return json_error(
_("Unable to access stream ({stream_name}).").format(
stream_name=unauthorized_streams[0].name, ))
# Newly created streams are also authorized for the creator
streams = authorized_streams + created_streams
if len(principals) > 0:
if user_profile.realm.is_zephyr_mirror_realm and not all(
stream.invite_only for stream in streams):
return json_error(
_("You can only invite other Zephyr mirroring users to private streams."
))
if not user_profile.can_subscribe_other_users():
if user_profile.realm.invite_to_stream_policy == Realm.POLICY_ADMINS_ONLY:
return json_error(
_("Only administrators can modify other users' subscriptions."
))
# Realm.POLICY_MEMBERS_ONLY only fails if the
# user is a guest, which happens in the decorator above.
assert user_profile.realm.invite_to_stream_policy == \
Realm.POLICY_FULL_MEMBERS_ONLY
return json_error(
_("Your account is too new to modify other users' subscriptions."
))
subscribers = {
principal_to_user_profile(user_profile, principal)
for principal in principals
}
else:
subscribers = {user_profile}
(subscribed,
already_subscribed) = bulk_add_subscriptions(streams,
subscribers,
acting_user=user_profile,
color_map=color_map)
# We can assume unique emails here for now, but we should eventually
# convert this function to be more id-centric.
email_to_user_profile: Dict[str, UserProfile] = {}
result: Dict[str, Any] = dict(subscribed=defaultdict(list),
already_subscribed=defaultdict(list))
for (subscriber, stream) in subscribed:
result["subscribed"][subscriber.email].append(stream.name)
email_to_user_profile[subscriber.email] = subscriber
for (subscriber, stream) in already_subscribed:
result["already_subscribed"][subscriber.email].append(stream.name)
bots = {subscriber.email: subscriber.is_bot for subscriber in subscribers}
newly_created_stream_names = {s.name for s in created_streams}
# Inform the user if someone else subscribed them to stuff,
# or if a new stream was created with the "announce" option.
notifications = []
if len(principals) > 0 and result["subscribed"]:
for email, subscribed_stream_names in result["subscribed"].items():
if email == user_profile.email:
# Don't send a Zulip if you invited yourself.
continue
if bots[email]:
# Don't send invitation Zulips to bots
continue
# For each user, we notify them about newly subscribed streams, except for
# streams that were newly created.
notify_stream_names = set(
subscribed_stream_names) - newly_created_stream_names
if not notify_stream_names:
continue
sender = get_system_bot(settings.NOTIFICATION_BOT)
recipient_user = email_to_user_profile[email]
msg = you_were_just_subscribed_message(
acting_user=user_profile,
recipient_user=recipient_user,
stream_names=notify_stream_names,
)
notifications.append(
internal_prep_private_message(realm=user_profile.realm,
sender=sender,
recipient_user=recipient_user,
content=msg))
if announce and len(created_streams) > 0:
notifications_stream = user_profile.realm.get_notifications_stream()
if notifications_stream is not None:
with override_language(
notifications_stream.realm.default_language):
if len(created_streams) > 1:
content = _(
"{user_name} created the following streams: {stream_str}."
)
else:
content = _(
"{user_name} created a new stream {stream_str}.")
topic = _('new streams')
content = content.format(
user_name=f"@_**{user_profile.full_name}|{user_profile.id}**",
stream_str=", ".join(f'#**{s.name}**'
for s in created_streams))
sender = get_system_bot(settings.NOTIFICATION_BOT)
notifications.append(
internal_prep_stream_message(
realm=user_profile.realm,
sender=sender,
stream=notifications_stream,
topic=topic,
content=content,
), )
if not user_profile.realm.is_zephyr_mirror_realm and len(
created_streams) > 0:
sender = get_system_bot(settings.NOTIFICATION_BOT)
for stream in created_streams:
with override_language(stream.realm.default_language):
notifications.append(
internal_prep_stream_message(
realm=user_profile.realm,
sender=sender,
stream=stream,
topic=Realm.STREAM_EVENTS_NOTIFICATION_TOPIC,
content=_('Stream created by {user_name}.').format(
user_name=
f"@_**{user_profile.full_name}|{user_profile.id}**",
),
), )
if len(notifications) > 0:
do_send_messages(notifications, mark_as_read=[user_profile.id])
result["subscribed"] = dict(result["subscribed"])
result["already_subscribed"] = dict(result["already_subscribed"])
if not authorization_errors_fatal:
result["unauthorized"] = [s.name for s in unauthorized_streams]
return json_success(result)
def consume(self, event: Dict[str, Any]) -> None:
if event['type'] == 'mark_stream_messages_as_read':
user_profile = get_user_profile_by_id(event['user_profile_id'])
client = Client.objects.get(id=event['client_id'])
for stream_id in event['stream_ids']:
# Since the user just unsubscribed, we don't require
# an active Subscription object (otherwise, private
# streams would never be accessible)
(stream, recipient, sub) = access_stream_by_id(user_profile, stream_id,
require_active=False)
do_mark_stream_messages_as_read(user_profile, client, stream)
elif event['type'] == 'clear_push_device_tokens':
try:
clear_push_device_tokens(event["user_profile_id"])
except PushNotificationBouncerRetryLaterError:
def failure_processor(event: Dict[str, Any]) -> None:
logger.warning(
"Maximum retries exceeded for trigger:%s event:clear_push_device_tokens",
event['user_profile_id'])
retry_event(self.queue_name, event, failure_processor)
elif event['type'] == 'realm_export':
start = time.time()
realm = Realm.objects.get(id=event['realm_id'])
output_dir = tempfile.mkdtemp(prefix="zulip-export-")
export_event = RealmAuditLog.objects.get(id=event['id'])
user_profile = get_user_profile_by_id(event['user_profile_id'])
try:
public_url = export_realm_wrapper(realm=realm, output_dir=output_dir,
threads=6, upload=True, public_only=True,
delete_after_upload=True)
except Exception:
export_event.extra_data = ujson.dumps(dict(
failed_timestamp=timezone_now().timestamp(),
))
export_event.save(update_fields=['extra_data'])
logging.error(
"Data export for %s failed after %s",
user_profile.realm.string_id, time.time() - start,
)
notify_realm_export(user_profile)
return
assert public_url is not None
# Update the extra_data field now that the export is complete.
export_event.extra_data = ujson.dumps(dict(
export_path=urllib.parse.urlparse(public_url).path,
))
export_event.save(update_fields=['extra_data'])
# Send a private message notification letting the user who
# triggered the export know the export finished.
with override_language(user_profile.default_language):
content = _("Your data export is complete and has been uploaded here:\n\n{public_url}").format(public_url=public_url)
internal_send_private_message(
realm=user_profile.realm,
sender=get_system_bot(settings.NOTIFICATION_BOT),
recipient_user=user_profile,
content=content,
)
# For future frontend use, also notify administrator
# clients that the export happened.
notify_realm_export(user_profile)
logging.info(
"Completed data export for %s in %s",
user_profile.realm.string_id, time.time() - start,
)
def post(self, request, format=None):
"""
"""
# Gather all the request data
username = request.POST.get('username')
email = request.POST.get('email')
password = request.POST.get('password')
contact_name = request.POST.get('contact_name')
activate = request.POST.get('activate', False)
org_manager = request.POST.get('org_manager', False)
send_email = request.POST.get('send_email', False)
language = request.POST.get('language', 'en')
conflicts = check_account_exists(email=email, username=username)
if conflicts:
return JsonResponse({'conflict_on_fields': conflicts}, status=409)
data = {
'username': username,
'email': email,
'password': password,
'name': contact_name,
}
# Go ahead and create the new user
with transaction.commit_on_success():
form = AccountCreationForm(
data=data,
tos_required=False,
)
(user, profile, registration) = _do_create_account(form)
create_comments_service_user(user)
if send_email:
with override_language(language):
context = {
'name': profile.name,
'key': registration.activation_key,
}
# composes activation email
subject = render_to_string('emails/activation_email_subject.txt', context)
subject = ''.join(subject.splitlines())
message = render_to_string('emails/activation_email.txt', context)
message_html = None
if (settings.FEATURES.get('ENABLE_MULTIPART_EMAIL')):
message_html = render_to_string('emails/html/activation_email.html', context)
from_address = microsite.get_value(
'email_from_address',
settings.DEFAULT_FROM_EMAIL
)
try:
mail.send_mail(subject, message, from_address, [user.email], html_message=message_html)
except Exception: # pylint: disable=broad-except
log.error(u'Unable to send activation email to remotely created user from "%s"', from_address, exc_info=True)
# Assing the user to the org management roles
if org_manager:
# We have to make them active for the roles to stick
user.is_active = True
user.save()
creator_role = OrgCourseCreatorRole(org_manager)
creator_role.add_users(user)
rerun_role = OrgRerunCreatorRole(org_manager)
rerun_role.add_users(user)
user.is_active = False
user.save()
if activate:
registration.activate()
return JsonResponse({"success": True}, status=201)
def test_locale(self):
with override_language('fr'):
self.assertEqual(text_type(StandardScopeClaims.info_profile[0]), 'Profil de base')
def _get_source_address(course_id,
course_title,
course_language,
truncate=True):
"""
Calculates an email address to be used as the 'from-address' for sent emails.
Makes a unique from name and address for each course, e.g.
"COURSE_TITLE" Course Staff <*****@*****.**>
If, when decoded to ascii, this from_addr is longer than 320 characters,
use the course_name rather than the course title, e.g.
"course_name" Course Staff <*****@*****.**>
The "truncate" kwarg is only used for tests.
"""
course_title_no_quotes = re.sub(r'"', '', course_title)
# For the email address, get the course. Then make sure that it can be used
# in an email address, by substituting a '_' anywhere a non-(ascii, period, or dash)
# character appears.
course_name = re.sub(r"[^\w.-]", '_', course_id.course)
# Use course.language if present
language = course_language if course_language else settings.LANGUAGE_CODE
with override_language(language):
# RFC2821 requires the byte order of the email address to be the name then email
# e.g. "John Doe <*****@*****.**>"
# Although the display will be flipped in RTL languages, the byte order is still the same.
from_addr_format = u'{name} {email}'.format(
# Translators: Bulk email from address e.g. ("Physics 101" Course Staff)
name=_(u'"{course_title}" Course Staff'),
email=
u'<{course_name}-{from_email}>', # xss-lint: disable=python-wrap-html
)
def format_address(course_title_no_quotes):
"""
Partial function for formatting the from_addr. Since
`course_title_no_quotes` may be truncated to make sure the returned
string has fewer than 320 characters, we define this function to make
it easy to determine quickly what the max length is for
`course_title_no_quotes`.
"""
return from_addr_format.format(
course_title=course_title_no_quotes,
course_name=course_name,
from_email=configuration_helpers.get_value(
'email_from_address', settings.BULK_EMAIL_DEFAULT_FROM_EMAIL))
from_addr = format_address(course_title_no_quotes)
# If the encoded from_addr is longer than 320 characters, reformat,
# but with the course name rather than course title.
# Amazon SES's from address field appears to have a maximum length of 320.
__, encoded_from_addr = forbid_multi_line_headers('from', from_addr,
'utf-8')
# It seems that this value is also escaped when set out to amazon, judging
# from our logs
escaped_encoded_from_addr = escape(encoded_from_addr)
if len(escaped_encoded_from_addr) >= 320 and truncate:
from_addr = format_address(course_name)
return from_addr
def build_email(template_prefix: str, to_user_ids: Optional[List[int]]=None,
to_emails: Optional[List[str]]=None, from_name: Optional[str]=None,
from_address: Optional[str]=None, reply_to_email: Optional[str]=None,
language: Optional[str]=None, context: Optional[Dict[str, Any]]=None
) -> EmailMultiAlternatives:
# Callers should pass exactly one of to_user_id and to_email.
assert (to_user_ids is None) ^ (to_emails is None)
if to_user_ids is not None:
to_users = [get_user_profile_by_id(to_user_id) for to_user_id in to_user_ids]
# Change to formataddr((to_user.full_name, to_user.email)) once
# https://github.com/zulip/zulip/issues/4676 is resolved
to_emails = [to_user.delivery_email for to_user in to_users]
if context is None:
context = {}
context.update({
'support_email': FromAddress.SUPPORT,
'email_images_base_uri': settings.ROOT_DOMAIN_URI + '/static/images/emails',
'physical_address': settings.PHYSICAL_ADDRESS,
})
def render_templates() -> Tuple[str, str, str]:
email_subject = loader.render_to_string(template_prefix + '.subject.txt',
context=context,
using='Jinja2_plaintext').strip().replace('\n', '')
message = loader.render_to_string(template_prefix + '.txt',
context=context, using='Jinja2_plaintext')
try:
html_message = loader.render_to_string(template_prefix + '.html', context)
except TemplateDoesNotExist:
emails_dir = os.path.dirname(template_prefix)
template = os.path.basename(template_prefix)
compiled_template_prefix = os.path.join(emails_dir, "compiled", template)
html_message = loader.render_to_string(compiled_template_prefix + '.html', context)
return (html_message, message, email_subject)
if not language and to_user_ids is not None:
language = to_users[0].default_language
if language:
with override_language(language):
# Make sure that we render the email using the target's native language
(html_message, message, email_subject) = render_templates()
else:
(html_message, message, email_subject) = render_templates()
logger.warning("Missing language for email template '{}'".format(template_prefix))
if from_name is None:
from_name = "Zulip"
if from_address is None:
from_address = FromAddress.NOREPLY
from_email = formataddr((from_name, from_address))
reply_to = None
if reply_to_email is not None:
reply_to = [reply_to_email]
# Remove the from_name in the reply-to for noreply emails, so that users
# see "noreply@..." rather than "Zulip" or whatever the from_name is
# when they reply in their email client.
elif from_address == FromAddress.NOREPLY:
reply_to = [FromAddress.NOREPLY]
mail = EmailMultiAlternatives(email_subject, message, from_email, to_emails, reply_to=reply_to)
if html_message is not None:
mail.attach_alternative(html_message, 'text/html')
return mail
def test_platform_language_is_used_for_logged_in_user(self):
with override_language('zh_CN'): # simulate a user login
subject, message = self.get_subject_and_message(None)
self.assertIn("You have been", subject)
self.assertIn("You have been", message)
def send_messages_for_new_subscribers(
user_profile: UserProfile,
subscribers: Set[UserProfile],
new_subscriptions: Dict[str, List[str]],
email_to_user_profile: Dict[str, UserProfile],
created_streams: List[Stream],
announce: bool,
) -> None:
"""
If you are subscribing lots of new users to new streams,
this function can be pretty expensive in terms of generating
lots of queries and sending lots of messages. We isolate
the code partly to make it easier to test things like
excessive query counts by mocking this function so that it
doesn't drown out query counts from other code.
"""
bots = {subscriber.email: subscriber.is_bot for subscriber in subscribers}
newly_created_stream_names = {s.name for s in created_streams}
# Inform the user if someone else subscribed them to stuff,
# or if a new stream was created with the "announce" option.
notifications = []
if new_subscriptions:
for email, subscribed_stream_names in new_subscriptions.items():
if email == user_profile.email:
# Don't send a Zulip if you invited yourself.
continue
if bots[email]:
# Don't send invitation Zulips to bots
continue
# For each user, we notify them about newly subscribed streams, except for
# streams that were newly created.
notify_stream_names = set(subscribed_stream_names) - newly_created_stream_names
if not notify_stream_names:
continue
sender = get_system_bot(settings.NOTIFICATION_BOT)
recipient_user = email_to_user_profile[email]
msg = you_were_just_subscribed_message(
acting_user=user_profile,
recipient_user=recipient_user,
stream_names=notify_stream_names,
)
notifications.append(
internal_prep_private_message(
realm=user_profile.realm,
sender=sender,
recipient_user=recipient_user,
content=msg,
)
)
if announce and len(created_streams) > 0:
notifications_stream = user_profile.realm.get_notifications_stream()
if notifications_stream is not None:
with override_language(notifications_stream.realm.default_language):
if len(created_streams) > 1:
content = _("{user_name} created the following streams: {stream_str}.")
else:
content = _("{user_name} created a new stream {stream_str}.")
topic = _("new streams")
content = content.format(
user_name=f"@_**{user_profile.full_name}|{user_profile.id}**",
stream_str=", ".join(f"#**{s.name}**" for s in created_streams),
)
sender = get_system_bot(settings.NOTIFICATION_BOT)
notifications.append(
internal_prep_stream_message(
sender=sender,
stream=notifications_stream,
topic=topic,
content=content,
),
)
if not user_profile.realm.is_zephyr_mirror_realm and len(created_streams) > 0:
sender = get_system_bot(settings.NOTIFICATION_BOT)
for stream in created_streams:
with override_language(stream.realm.default_language):
notifications.append(
internal_prep_stream_message(
sender=sender,
stream=stream,
topic=Realm.STREAM_EVENTS_NOTIFICATION_TOPIC,
content=_("Stream created by {user_name}.").format(
user_name=f"@_**{user_profile.full_name}|{user_profile.id}**",
),
),
)
if len(notifications) > 0:
do_send_messages(notifications, mark_as_read=[user_profile.id])
def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile,
missed_messages: List[Dict[
str, Any]],
message_count: int) -> None:
"""
Send a reminder email to a user if she's missed some PMs by being offline.
The email will have its reply to address set to a limited used email
address that will send a Zulip message to the correct recipient. This
allows the user to respond to missed PMs, huddles, and @-mentions directly
from the email.
`user_profile` is the user to send the reminder to
`missed_messages` is a list of dictionaries to Message objects and other data
for a group of messages that share a recipient (and topic)
"""
from zerver.context_processors import common_context
recipients = {(msg["message"].recipient_id, msg["message"].topic_name())
for msg in missed_messages}
if len(recipients) != 1:
raise ValueError(
f"All missed_messages must have the same recipient and topic {recipients!r}",
)
# This link is no longer a part of the email, but keeping the code in case
# we find a clean way to add it back in the future
unsubscribe_link = one_click_unsubscribe_link(user_profile,
"missed_messages")
context = common_context(user_profile)
context.update(
name=user_profile.full_name,
message_count=message_count,
unsubscribe_link=unsubscribe_link,
realm_name_in_notifications=user_profile.realm_name_in_notifications,
)
mentioned_user_group_name = get_mentioned_user_group_name(
missed_messages, user_profile)
triggers = [message["trigger"] for message in missed_messages]
unique_triggers = set(triggers)
context.update(
mention="mentioned" in unique_triggers
or "wildcard_mentioned" in unique_triggers,
stream_email_notify="stream_email_notify" in unique_triggers,
mention_count=triggers.count("mentioned") +
triggers.count("wildcard_mentioned"),
mentioned_user_group_name=mentioned_user_group_name,
)
# If this setting (email mirroring integration) is enabled, only then
# can users reply to email to send message to Zulip. Thus, one must
# ensure to display warning in the template.
if settings.EMAIL_GATEWAY_PATTERN:
context.update(reply_to_zulip=True, )
else:
context.update(reply_to_zulip=False, )
from zerver.lib.email_mirror import create_missed_message_address
reply_to_address = create_missed_message_address(
user_profile, missed_messages[0]["message"])
if reply_to_address == FromAddress.NOREPLY:
reply_to_name = ""
else:
reply_to_name = "Zulip"
narrow_url = get_narrow_url(user_profile, missed_messages[0]["message"])
context.update(narrow_url=narrow_url, )
senders = list({m["message"].sender for m in missed_messages})
if missed_messages[0]["message"].recipient.type == Recipient.HUDDLE:
display_recipient = get_display_recipient(
missed_messages[0]["message"].recipient)
# Make sure that this is a list of strings, not a string.
assert not isinstance(display_recipient, str)
other_recipients = [
r["full_name"] for r in display_recipient
if r["id"] != user_profile.id
]
context.update(group_pm=True)
if len(other_recipients) == 2:
huddle_display_name = " and ".join(other_recipients)
context.update(huddle_display_name=huddle_display_name)
elif len(other_recipients) == 3:
huddle_display_name = (
f"{other_recipients[0]}, {other_recipients[1]}, and {other_recipients[2]}"
)
context.update(huddle_display_name=huddle_display_name)
else:
huddle_display_name = "{}, and {} others".format(
", ".join(other_recipients[:2]),
len(other_recipients) - 2)
context.update(huddle_display_name=huddle_display_name)
elif missed_messages[0]["message"].recipient.type == Recipient.PERSONAL:
context.update(private_message=True)
elif context["mention"] or context["stream_email_notify"]:
# Keep only the senders who actually mentioned the user
if context["mention"]:
senders = list({
m["message"].sender
for m in missed_messages if m["trigger"] == "mentioned"
or m["trigger"] == "wildcard_mentioned"
})
message = missed_messages[0]["message"]
stream = Stream.objects.only("id",
"name").get(id=message.recipient.type_id)
stream_header = f"{stream.name} > {message.topic_name()}"
context.update(stream_header=stream_header, )
else:
raise AssertionError("Invalid messages!")
# If message content is disabled, then flush all information we pass to email.
if not message_content_allowed_in_missedmessage_emails(user_profile):
realm = user_profile.realm
context.update(
reply_to_zulip=False,
messages=[],
sender_str="",
realm_str=realm.name,
huddle_display_name="",
show_message_content=False,
message_content_disabled_by_user=not user_profile.
message_content_in_email_notifications,
message_content_disabled_by_realm=not realm.
message_content_allowed_in_email_notifications,
)
else:
context.update(
messages=build_message_list(
user=user_profile,
messages=[m["message"] for m in missed_messages],
stream_map={},
),
sender_str=", ".join(sender.full_name for sender in senders),
realm_str=user_profile.realm.name,
show_message_content=True,
)
with override_language(user_profile.default_language):
from_name: str = _("Zulip notifications")
from_address = FromAddress.NOREPLY
if len(senders) == 1 and settings.SEND_MISSED_MESSAGE_EMAILS_AS_USER:
# If this setting is enabled, you can reply to the Zulip
# message notification emails directly back to the original sender.
# However, one must ensure the Zulip server is in the SPF
# record for the domain, or there will be spam/deliverability
# problems.
#
# Also, this setting is not really compatible with
# EMAIL_ADDRESS_VISIBILITY_ADMINS.
sender = senders[0]
from_name, from_address = (sender.full_name, sender.email)
context.update(reply_to_zulip=False, )
email_dict = {
"template_prefix":
"zerver/emails/missed_message",
"to_user_ids": [user_profile.id],
"from_name":
from_name,
"from_address":
from_address,
"reply_to_email":
str(Address(display_name=reply_to_name, addr_spec=reply_to_address)),
"context":
context,
}
queue_json_publish("email_senders", email_dict)
user_profile.last_reminder = timezone_now()
user_profile.save(update_fields=["last_reminder"])
def consume(self, event: Dict[str, Any]) -> None:
start = time.time()
if event["type"] == "mark_stream_messages_as_read":
user_profile = get_user_profile_by_id(event["user_profile_id"])
for recipient_id in event["stream_recipient_ids"]:
do_mark_stream_messages_as_read(user_profile, recipient_id)
elif event["type"] == "mark_stream_messages_as_read_for_everyone":
# This event is generated by the stream deactivation code path.
batch_size = 100
offset = 0
while True:
messages = Message.objects.filter(
recipient_id=event["stream_recipient_id"]).order_by(
"id")[offset:offset + batch_size]
UserMessage.objects.filter(message__in=messages).extra(
where=[UserMessage.where_unread()]).update(
flags=F("flags").bitor(UserMessage.flags.read))
offset += len(messages)
if len(messages) < batch_size:
break
elif event["type"] == "clear_push_device_tokens":
try:
clear_push_device_tokens(event["user_profile_id"])
except PushNotificationBouncerRetryLaterError:
def failure_processor(event: Dict[str, Any]) -> None:
logger.warning(
"Maximum retries exceeded for trigger:%s event:clear_push_device_tokens",
event["user_profile_id"],
)
retry_event(self.queue_name, event, failure_processor)
elif event["type"] == "realm_export":
realm = Realm.objects.get(id=event["realm_id"])
output_dir = tempfile.mkdtemp(prefix="zulip-export-")
export_event = RealmAuditLog.objects.get(id=event["id"])
user_profile = get_user_profile_by_id(event["user_profile_id"])
try:
public_url = export_realm_wrapper(
realm=realm,
output_dir=output_dir,
threads=6,
upload=True,
public_only=True,
delete_after_upload=True,
)
except Exception:
export_event.extra_data = orjson.dumps(
dict(failed_timestamp=timezone_now().timestamp(),
)).decode()
export_event.save(update_fields=["extra_data"])
logging.error(
"Data export for %s failed after %s",
user_profile.realm.string_id,
time.time() - start,
)
notify_realm_export(user_profile)
return
assert public_url is not None
# Update the extra_data field now that the export is complete.
export_event.extra_data = orjson.dumps(
dict(export_path=urllib.parse.urlparse(public_url).path,
)).decode()
export_event.save(update_fields=["extra_data"])
# Send a private message notification letting the user who
# triggered the export know the export finished.
with override_language(user_profile.default_language):
content = _(
"Your data export is complete and has been uploaded here:\n\n{public_url}"
).format(public_url=public_url)
internal_send_private_message(
sender=get_system_bot(settings.NOTIFICATION_BOT),
recipient_user=user_profile,
content=content,
)
# For future frontend use, also notify administrator
# clients that the export happened.
notify_realm_export(user_profile)
logging.info(
"Completed data export for %s in %s",
user_profile.realm.string_id,
time.time() - start,
)
end = time.time()
logger.info("deferred_work processed %s event (%dms)", event["type"],
(end - start) * 1000)
