def create_user_profile(realm: Realm, email: str, password: Optional[str],
active: bool, bot_type: Optional[int], full_name: str,
short_name: str, bot_owner: Optional[UserProfile],
is_mirror_dummy: bool, tos_version: Optional[str],
timezone: Optional[str],
tutorial_status: Optional[str] = UserProfile.TUTORIAL_WAITING,
enter_sends: bool = False) -> UserProfile:
now = timezone_now()
email = UserManager.normalize_email(email)
user_profile = UserProfile(email=email, is_staff=False, is_active=active,
full_name=full_name, short_name=short_name,
last_login=now, date_joined=now, realm=realm,
pointer=-1, is_bot=bool(bot_type), bot_type=bot_type,
bot_owner=bot_owner, is_mirror_dummy=is_mirror_dummy,
tos_version=tos_version, timezone=timezone,
tutorial_status=tutorial_status,
enter_sends=enter_sends,
onboarding_steps=ujson.dumps([]),
default_language=realm.default_language,
twenty_four_hour_time=realm.default_twenty_four_hour_time,
delivery_email=email)
if bot_type or not active:
password = None
user_profile.set_password(password)
user_profile.api_key = generate_api_key()
return user_profile
def create_user_profile(realm, email, password, active, bot_type, full_name,
short_name, bot_owner, is_mirror_dummy, tos_version,
timezone, tutorial_status=UserProfile.TUTORIAL_WAITING,
enter_sends=False):
# type: (Realm, Text, Optional[Text], bool, Optional[int], Text, Text, Optional[UserProfile], bool, Text, Optional[Text], Optional[Text], bool) -> UserProfile
now = timezone_now()
email = UserManager.normalize_email(email)
user_profile = UserProfile(email=email, is_staff=False, is_active=active,
full_name=full_name, short_name=short_name,
last_login=now, date_joined=now, realm=realm,
pointer=-1, is_bot=bool(bot_type), bot_type=bot_type,
bot_owner=bot_owner, is_mirror_dummy=is_mirror_dummy,
tos_version=tos_version, timezone=timezone,
tutorial_status=tutorial_status,
enter_sends=enter_sends,
onboarding_steps=ujson.dumps([]),
default_language=realm.default_language)
if bot_type or not active:
password = None
user_profile.set_password(password)
user_profile.api_key = random_api_key()
return user_profile
def do_create_customer(user: UserProfile, stripe_token: Optional[str]=None,
coupon: Optional[Coupon]=None) -> stripe.Customer:
realm = user.realm
stripe_coupon_id = None
if coupon is not None:
stripe_coupon_id = coupon.stripe_coupon_id
# We could do a better job of handling race conditions here, but if two
# people from a realm try to upgrade at exactly the same time, the main
# bad thing that will happen is that we will create an extra stripe
# customer that we can delete or ignore.
stripe_customer = stripe.Customer.create(
description="%s (%s)" % (realm.string_id, realm.name),
email=user.email,
metadata={'realm_id': realm.id, 'realm_str': realm.string_id},
source=stripe_token,
coupon=stripe_coupon_id)
if PRINT_STRIPE_FIXTURE_DATA:
print(''.join(['"create_customer": ', str(stripe_customer), ','])) # nocoverage
event_time = timestamp_to_datetime(stripe_customer.created)
with transaction.atomic():
RealmAuditLog.objects.create(
realm=user.realm, acting_user=user, event_type=RealmAuditLog.STRIPE_CUSTOMER_CREATED,
event_time=event_time)
if stripe_token is not None:
RealmAuditLog.objects.create(
realm=user.realm, acting_user=user, event_type=RealmAuditLog.STRIPE_CARD_CHANGED,
event_time=event_time)
Customer.objects.create(realm=realm, stripe_customer_id=stripe_customer.id)
user.is_billing_admin = True
user.save(update_fields=["is_billing_admin"])
return stripe_customer
def create_user_profile(realm: Realm, email: str, password: Optional[str],
active: bool, bot_type: Optional[int], full_name: str,
short_name: str, bot_owner: Optional[UserProfile],
is_mirror_dummy: bool, tos_version: Optional[str],
timezone: Optional[str],
tutorial_status: Optional[str] = UserProfile.TUTORIAL_WAITING,
enter_sends: bool = False) -> UserProfile:
now = timezone_now()
email = UserManager.normalize_email(email)
user_profile = UserProfile(is_staff=False, is_active=active,
full_name=full_name, short_name=short_name,
last_login=now, date_joined=now, realm=realm,
pointer=-1, is_bot=bool(bot_type), bot_type=bot_type,
bot_owner=bot_owner, is_mirror_dummy=is_mirror_dummy,
tos_version=tos_version, timezone=timezone,
tutorial_status=tutorial_status,
enter_sends=enter_sends,
onboarding_steps=ujson.dumps([]),
default_language=realm.default_language,
twenty_four_hour_time=realm.default_twenty_four_hour_time,
delivery_email=email)
if bot_type or not active:
password = None
if realm.email_address_visibility == Realm.EMAIL_ADDRESS_VISIBILITY_EVERYONE:
# If emails are visible to everyone, we can set this here and save a DB query
user_profile.email = get_display_email_address(user_profile, realm)
user_profile.set_password(password)
user_profile.api_key = generate_api_key()
return user_profile
def create_user_profile(realm, email, password, active, bot_type, full_name,
short_name, bot_owner, is_mirror_dummy, tos_version):
# type: (Realm, text_type, text_type, bool, Optional[int], text_type, text_type, Optional[UserProfile], bool, Optional[text_type]) -> UserProfile
now = timezone.now()
email = UserManager.normalize_email(email)
enable_stream_desktop_notifications = (realm.domain != 'zulip.com')
user_profile = UserProfile(email=email, is_staff=False, is_active=active,
full_name=full_name, short_name=short_name,
last_login=now, date_joined=now, realm=realm,
pointer=-1, is_bot=bool(bot_type), bot_type=bot_type,
is_mirror_dummy=is_mirror_dummy,
tos_version=tos_version,
enable_stream_desktop_notifications=enable_stream_desktop_notifications,
onboarding_steps=ujson.dumps([]),
default_language=realm.default_language)
if bot_owner is not None:
# `user_profile.bot_owner = bot_owner` doesn't work on python 3.4
user_profile.bot_owner_id = bot_owner.id
if bot_type or not active:
password = None
user_profile.set_password(password)
user_profile.api_key = random_api_key()
return user_profile
def get_next_hotspots(user: UserProfile) -> List[Dict[str, object]]:
# For manual testing, it can be convenient to set
# ALWAYS_SEND_ALL_HOTSPOTS=True in `zproject/dev_settings.py` to
# make it easy to click on all of the hotspots.
if settings.ALWAYS_SEND_ALL_HOTSPOTS:
return [{
'name': hotspot,
'title': ALL_HOTSPOTS[hotspot]['title'],
'description': ALL_HOTSPOTS[hotspot]['description'],
'delay': 0,
} for hotspot in ALL_HOTSPOTS]
if user.tutorial_status == UserProfile.TUTORIAL_FINISHED:
return []
seen_hotspots = frozenset(UserHotspot.objects.filter(user=user).values_list('hotspot', flat=True))
for hotspot in ['intro_reply', 'intro_streams', 'intro_topics', 'intro_compose']:
if hotspot not in seen_hotspots:
return [{
'name': hotspot,
'title': ALL_HOTSPOTS[hotspot]['title'],
'description': ALL_HOTSPOTS[hotspot]['description'],
'delay': 0.5,
}]
user.tutorial_status = UserProfile.TUTORIAL_FINISHED
user.save(update_fields=['tutorial_status'])
return []
def do_create_stripe_customer(user: UserProfile, stripe_token: Optional[str]=None) -> Customer:
realm = user.realm
# We could do a better job of handling race conditions here, but if two
# people from a realm try to upgrade at exactly the same time, the main
# bad thing that will happen is that we will create an extra stripe
# customer that we can delete or ignore.
stripe_customer = stripe.Customer.create(
description="%s (%s)" % (realm.string_id, realm.name),
email=user.email,
metadata={'realm_id': realm.id, 'realm_str': realm.string_id},
source=stripe_token)
event_time = timestamp_to_datetime(stripe_customer.created)
with transaction.atomic():
RealmAuditLog.objects.create(
realm=user.realm, acting_user=user, event_type=RealmAuditLog.STRIPE_CUSTOMER_CREATED,
event_time=event_time)
if stripe_token is not None:
RealmAuditLog.objects.create(
realm=user.realm, acting_user=user, event_type=RealmAuditLog.STRIPE_CARD_CHANGED,
event_time=event_time)
customer, created = Customer.objects.update_or_create(realm=realm, defaults={
'stripe_customer_id': stripe_customer.id})
user.is_billing_admin = True
user.save(update_fields=["is_billing_admin"])
return customer
def copy_hotpots(source_profile: UserProfile, target_profile: UserProfile) -> None:
for userhotspot in frozenset(UserHotspot.objects.filter(user=source_profile)):
UserHotspot.objects.create(user=target_profile, hotspot=userhotspot.hotspot,
timestamp=userhotspot.timestamp)
target_profile.tutorial_status = source_profile.tutorial_status
target_profile.onboarding_steps = source_profile.onboarding_steps
target_profile.save(update_fields=['tutorial_status', 'onboarding_steps'])
def set_tutorial_status(request: HttpRequest, user_profile: UserProfile,
status: str=REQ(validator=check_string)) -> HttpResponse:
if status == 'started':
user_profile.tutorial_status = UserProfile.TUTORIAL_STARTED
elif status == 'finished':
user_profile.tutorial_status = UserProfile.TUTORIAL_FINISHED
user_profile.save(update_fields=["tutorial_status"])
return json_success()
def do_soft_deactivate_user(user_profile: UserProfile) -> None:
user_profile.last_active_message_id = UserMessage.objects.filter(
user_profile=user_profile).order_by(
'-message__id')[0].message_id
user_profile.long_term_idle = True
user_profile.save(update_fields=[
'long_term_idle',
'last_active_message_id'])
logger.info('Soft Deactivated user %s (%s)' %
(user_profile.id, user_profile.email))
def do_soft_deactivate_user(user_profile: UserProfile) -> None:
try:
user_profile.last_active_message_id = UserMessage.objects.filter(
user_profile=user_profile).order_by(
'-message__id')[0].message_id
except IndexError: # nocoverage
# In the unlikely event that a user somehow has never received
# a message, we just use the overall max message ID.
user_profile.last_active_message_id = Message.objects.max().id
user_profile.long_term_idle = True
user_profile.save(update_fields=[
'long_term_idle',
'last_active_message_id'])
logger.info('Soft Deactivated user %s (%s)' %
(user_profile.id, user_profile.email))
def maybe_catch_up_soft_deactivated_user(user_profile: UserProfile) -> Union[UserProfile, None]:
if user_profile.long_term_idle:
add_missing_messages(user_profile)
user_profile.long_term_idle = False
user_profile.save(update_fields=['long_term_idle'])
RealmAuditLog.objects.create(
realm=user_profile.realm,
modified_user=user_profile,
event_type=RealmAuditLog.USER_SOFT_ACTIVATED,
event_time=timezone_now()
)
logger.info('Soft Reactivated user %s (%s)' %
(user_profile.id, user_profile.email))
return user_profile
return None
def list_to_streams(streams_raw: Iterable[Mapping[str, Any]],
user_profile: UserProfile,
autocreate: bool=False) -> Tuple[List[Stream], List[Stream]]:
"""Converts list of dicts to a list of Streams, validating input in the process
For each stream name, we validate it to ensure it meets our
requirements for a proper stream name using check_stream_name.
This function in autocreate mode should be atomic: either an exception will be raised
during a precheck, or all the streams specified will have been created if applicable.
@param streams_raw The list of stream dictionaries to process;
names should already be stripped of whitespace by the caller.
@param user_profile The user for whom we are retreiving the streams
@param autocreate Whether we should create streams if they don't already exist
"""
# Validate all streams, getting extant ones, then get-or-creating the rest.
stream_set = set(stream_dict["name"] for stream_dict in streams_raw)
for stream_name in stream_set:
# Stream names should already have been stripped by the
# caller, but it makes sense to verify anyway.
assert stream_name == stream_name.strip()
check_stream_name(stream_name)
existing_streams = [] # type: List[Stream]
missing_stream_dicts = [] # type: List[Mapping[str, Any]]
existing_stream_map = bulk_get_streams(user_profile.realm, stream_set)
for stream_dict in streams_raw:
stream_name = stream_dict["name"]
stream = existing_stream_map.get(stream_name.lower())
if stream is None:
missing_stream_dicts.append(stream_dict)
else:
existing_streams.append(stream)
if len(missing_stream_dicts) == 0:
# This is the happy path for callers who expected all of these
# streams to exist already.
created_streams = [] # type: List[Stream]
else:
# autocreate=True path starts here
if not user_profile.can_create_streams():
raise JsonableError(_('User cannot create streams.'))
elif not autocreate:
raise JsonableError(_("Stream(s) (%s) do not exist") % ", ".join(
stream_dict["name"] for stream_dict in missing_stream_dicts))
# We already filtered out existing streams, so dup_streams
# will normally be an empty list below, but we protect against somebody
# else racing to create the same stream. (This is not an entirely
# paranoid approach, since often on Zulip two people will discuss
# creating a new stream, and both people eagerly do it.)
created_streams, dup_streams = create_streams_if_needed(realm=user_profile.realm,
stream_dicts=missing_stream_dicts)
existing_streams += dup_streams
return existing_streams, created_streams
def update_display_settings_backend(
request: HttpRequest, user_profile: UserProfile,
twenty_four_hour_time: Optional[bool]=REQ(validator=check_bool, default=None),
dense_mode: Optional[bool]=REQ(validator=check_bool, default=None),
starred_message_counts: Optional[bool]=REQ(validator=check_bool, default=None),
high_contrast_mode: Optional[bool]=REQ(validator=check_bool, default=None),
night_mode: Optional[bool]=REQ(validator=check_bool, default=None),
translate_emoticons: Optional[bool]=REQ(validator=check_bool, default=None),
default_language: Optional[bool]=REQ(validator=check_string, default=None),
left_side_userlist: Optional[bool]=REQ(validator=check_bool, default=None),
emojiset: Optional[str]=REQ(validator=check_string, default=None),
timezone: Optional[str]=REQ(validator=check_string, default=None)) -> HttpResponse:
if (default_language is not None and
default_language not in get_available_language_codes()):
raise JsonableError(_("Invalid language '%s'" % (default_language,)))
if (timezone is not None and
timezone not in get_all_timezones()):
raise JsonableError(_("Invalid timezone '%s'" % (timezone,)))
if (emojiset is not None and
emojiset not in UserProfile.emojiset_choices()):
raise JsonableError(_("Invalid emojiset '%s'" % (emojiset,)))
request_settings = {k: v for k, v in list(locals().items()) if k in user_profile.property_types}
result = {} # type: Dict[str, Any]
for k, v in list(request_settings.items()):
if v is not None and getattr(user_profile, k) != v:
do_set_user_display_setting(user_profile, k, v)
result[k] = v
return json_success(result)
def _deactivate_user_profile_backend(request: HttpRequest, user_profile: UserProfile,
target: UserProfile) -> HttpResponse:
if not user_profile.can_admin_user(target):
return json_error(_('Insufficient permission'))
do_deactivate_user(target, acting_user=user_profile)
return json_success()
def copy_user_settings(source_profile: UserProfile, target_profile: UserProfile) -> None:
"""Warning: Does not save, to avoid extra database queries"""
for settings_name in UserProfile.property_types:
value = getattr(source_profile, settings_name)
setattr(target_profile, settings_name, value)
for settings_name in UserProfile.notification_setting_types:
value = getattr(source_profile, settings_name)
setattr(target_profile, settings_name, value)
setattr(target_profile, "full_name", source_profile.full_name)
target_profile.save()
if source_profile.avatar_source == UserProfile.AVATAR_FROM_USER:
from zerver.lib.actions import do_change_avatar_fields
do_change_avatar_fields(target_profile, UserProfile.AVATAR_FROM_USER)
copy_avatar(source_profile, target_profile)
def access_bot_by_id(user_profile: UserProfile, user_id: int) -> UserProfile:
try:
target = get_user_profile_by_id_in_realm(user_id, user_profile.realm)
except UserProfile.DoesNotExist:
raise JsonableError(_("No such bot"))
if not target.is_bot:
raise JsonableError(_("No such bot"))
if not user_profile.can_admin_user(target):
raise JsonableError(_("Insufficient permission"))
return target
def test(user: UserProfile) -> bool:
"""
:if_configured: If ``True``, an authenticated user with no confirmed
OTP devices will be allowed. Default is ``False``. If ``False``,
2FA will not do any authentication.
"""
if_configured = settings.TWO_FACTOR_AUTHENTICATION_ENABLED
if not if_configured:
return True
return user.is_verified() or (_user_is_authenticated(user)
and not user_has_device(user))
def gather_new_users(user_profile: UserProfile, threshold: datetime.datetime) -> Tuple[int, List[str]]:
# Gather information on users in the realm who have recently
# joined.
if not user_profile.can_access_all_realm_members():
new_users = [] # type: List[UserProfile]
else:
new_users = list(UserProfile.objects.filter(
realm=user_profile.realm, date_joined__gt=threshold,
is_bot=False))
user_names = [user.full_name for user in new_users]
return len(user_names), user_names
def reactivate_user_backend(request: HttpRequest, user_profile: UserProfile,
email: Text) -> HttpResponse:
try:
target = get_user(email, user_profile.realm)
except UserProfile.DoesNotExist:
return json_error(_('No such user'))
if not user_profile.can_admin_user(target):
return json_error(_('Insufficient permission'))
do_reactivate_user(target, acting_user=user_profile)
return json_success()
def access_user_by_id(user_profile: UserProfile, user_id: int,
allow_deactivated: bool=False, allow_bots: bool=False) -> UserProfile:
try:
target = get_user_profile_by_id_in_realm(user_id, user_profile.realm)
except UserProfile.DoesNotExist:
raise JsonableError(_("No such user"))
if target.is_bot and not allow_bots:
raise JsonableError(_("No such user"))
if not target.is_active and not allow_deactivated:
raise JsonableError(_("User is deactivated"))
if not user_profile.can_admin_user(target):
raise JsonableError(_("Insufficient permission"))
return target
def create_user_profile(realm, email, password, active, bot, full_name,
short_name, bot_owner, is_mirror_dummy):
now = timezone.now()
email = UserManager.normalize_email(email)
enable_stream_desktop_notifications = (realm.domain != 'zulip.com')
user_profile = UserProfile(email=email, is_staff=False, is_active=active,
full_name=full_name, short_name=short_name,
last_login=now, date_joined=now, realm=realm,
pointer=-1, is_bot=bot, bot_owner=bot_owner,
is_mirror_dummy=is_mirror_dummy,
enable_stream_desktop_notifications=enable_stream_desktop_notifications,
onboarding_steps=ujson.dumps([]))
if bot or not active:
password = None
user_profile.set_password(password)
user_profile.api_key = random_api_key()
return user_profile
def regenerate_bot_api_key(request: HttpRequest, user_profile: UserProfile, email: Text) -> HttpResponse:
try:
bot = get_user(email, user_profile.realm)
except UserProfile.DoesNotExist:
return json_error(_('No such user'))
if not user_profile.can_admin_user(bot):
return json_error(_('Insufficient permission'))
do_regenerate_api_key(bot, user_profile)
json_result = dict(
api_key = bot.api_key
)
return json_success(json_result)
def gather_new_streams(user_profile: UserProfile,
threshold: datetime.datetime) -> Tuple[int, Dict[str, List[str]]]:
if user_profile.can_access_public_streams():
new_streams = list(get_active_streams(user_profile.realm).filter(
invite_only=False, date_created__gt=threshold))
else:
new_streams = []
base_url = "%s/#narrow/stream/" % (user_profile.realm.uri,)
streams_html = []
streams_plain = []
for stream in new_streams:
narrow_url = base_url + encode_stream(stream.id, stream.name)
stream_link = "<a href='%s'>%s</a>" % (narrow_url, stream.name)
streams_html.append(stream_link)
streams_plain.append(stream.name)
return len(new_streams), {"html": streams_html, "plain": streams_plain}
def update_user_backend(request: HttpRequest, user_profile: UserProfile, email: Text,
full_name: Optional[Text]=REQ(default="", validator=check_string),
is_admin: Optional[bool]=REQ(default=None, validator=check_bool)) -> HttpResponse:
try:
target = get_user(email, user_profile.realm)
except UserProfile.DoesNotExist:
return json_error(_('No such user'))
if not user_profile.can_admin_user(target):
return json_error(_('Insufficient permission'))
if is_admin is not None:
if not is_admin and check_last_admin(user_profile):
return json_error(_('Cannot remove the only organization administrator'))
do_change_is_admin(target, is_admin)
if (full_name is not None and target.full_name != full_name and
full_name.strip() != ""):
# We don't respect `name_changes_disabled` here because the request
# is on behalf of the administrator.
check_change_full_name(target, full_name, user_profile)
return json_success()
def get_events_backend(
request: HttpRequest,
user_profile: UserProfile,
# user_client is intended only for internal Django=>Tornado requests
# and thus shouldn't be documented for external use.
user_client: Optional[Client] = REQ(converter=get_client,
default=None,
intentionally_undocumented=True),
last_event_id: Optional[int] = REQ(converter=int, default=None),
queue_id: Optional[str] = REQ(default=None),
# apply_markdown, client_gravatar, all_public_streams, and various
# other parameters are only used when registering a new queue via this
# endpoint. This is a feature used primarily by get_events_internal
# and not expected to be used by third-party clients.
apply_markdown: bool = REQ(default=False,
json_validator=check_bool,
intentionally_undocumented=True),
client_gravatar: bool = REQ(default=False,
json_validator=check_bool,
intentionally_undocumented=True),
slim_presence: bool = REQ(default=False,
json_validator=check_bool,
intentionally_undocumented=True),
all_public_streams: bool = REQ(default=False,
json_validator=check_bool,
intentionally_undocumented=True),
event_types: Optional[Sequence[str]] = REQ(
default=None,
json_validator=check_list(check_string),
intentionally_undocumented=True),
dont_block: bool = REQ(default=False, json_validator=check_bool),
narrow: Sequence[Sequence[str]] = REQ(
default=[],
json_validator=check_list(check_list(check_string)),
intentionally_undocumented=True,
),
lifespan_secs: int = REQ(default=0,
converter=to_non_negative_int,
intentionally_undocumented=True),
bulk_message_deletion: bool = REQ(default=False,
json_validator=check_bool,
intentionally_undocumented=True),
stream_typing_notifications: bool = REQ(default=False,
json_validator=check_bool,
intentionally_undocumented=True),
) -> HttpResponse:
if all_public_streams and not user_profile.can_access_public_streams():
raise JsonableError(_("User not authorized for this query"))
# Extract the Tornado handler from the request
handler: AsyncDjangoHandler = request._tornado_handler
if user_client is None:
valid_user_client = request.client
else:
valid_user_client = user_client
events_query = dict(
user_profile_id=user_profile.id,
queue_id=queue_id,
last_event_id=last_event_id,
event_types=event_types,
client_type_name=valid_user_client.name,
all_public_streams=all_public_streams,
lifespan_secs=lifespan_secs,
narrow=narrow,
dont_block=dont_block,
handler_id=handler.handler_id,
)
if queue_id is None:
events_query["new_queue_data"] = dict(
user_profile_id=user_profile.id,
realm_id=user_profile.realm_id,
event_types=event_types,
client_type_name=valid_user_client.name,
apply_markdown=apply_markdown,
client_gravatar=client_gravatar,
slim_presence=slim_presence,
all_public_streams=all_public_streams,
queue_timeout=lifespan_secs,
last_connection_time=time.time(),
narrow=narrow,
bulk_message_deletion=bulk_message_deletion,
stream_typing_notifications=stream_typing_notifications,
)
result = fetch_events(events_query)
if "extra_log_data" in result:
request._log_data["extra"] = result["extra_log_data"]
if result["type"] == "async":
# Mark this response with .asynchronous; this will result in
# Tornado discarding the response and instead long-polling the
# request. See zulip_finish for more design details.
handler._request = request
response = json_success()
response.asynchronous = True
return response
if result["type"] == "error":
raise result["exception"]
return json_success(result["response"])
def validation_func(user_profile: UserProfile) -> bool:
user_profile.refresh_from_db()
return user_profile.can_add_custom_emoji()
def add_subscriptions_backend(
request: HttpRequest,
user_profile: UserProfile,
streams_raw: Iterable[Mapping[str, str]] = REQ(
"subscriptions", json_validator=add_subscriptions_schema
),
invite_only: bool = REQ(json_validator=check_bool, default=False),
stream_post_policy: int = REQ(
json_validator=check_int_in(Stream.STREAM_POST_POLICY_TYPES),
default=Stream.STREAM_POST_POLICY_EVERYONE,
),
history_public_to_subscribers: Optional[bool] = REQ(json_validator=check_bool, default=None),
message_retention_days: Union[str, int] = REQ(
json_validator=check_string_or_int, default=RETENTION_DEFAULT
),
announce: bool = REQ(json_validator=check_bool, default=False),
principals: Union[Sequence[str], Sequence[int]] = REQ(
json_validator=check_principals,
default=EMPTY_PRINCIPALS,
),
authorization_errors_fatal: bool = REQ(json_validator=check_bool, default=True),
) -> HttpResponse:
realm = user_profile.realm
stream_dicts = []
color_map = {}
for stream_dict in streams_raw:
# 'color' field is optional
# check for its presence in the streams_raw first
if "color" in stream_dict:
color_map[stream_dict["name"]] = stream_dict["color"]
stream_dict_copy: StreamDict = {}
stream_dict_copy["name"] = stream_dict["name"].strip()
# We don't allow newline characters in stream descriptions.
if "description" in stream_dict:
stream_dict_copy["description"] = stream_dict["description"].replace("\n", " ")
stream_dict_copy["invite_only"] = invite_only
stream_dict_copy["stream_post_policy"] = stream_post_policy
stream_dict_copy["history_public_to_subscribers"] = history_public_to_subscribers
stream_dict_copy["message_retention_days"] = parse_message_retention_days(
message_retention_days, Stream.MESSAGE_RETENTION_SPECIAL_VALUES_MAP
)
stream_dicts.append(stream_dict_copy)
# Validation of the streams arguments, including enforcement of
# can_create_streams policy and check_stream_name policy is inside
# list_to_streams.
existing_streams, created_streams = list_to_streams(stream_dicts, user_profile, autocreate=True)
authorized_streams, unauthorized_streams = filter_stream_authorization(
user_profile, existing_streams
)
if len(unauthorized_streams) > 0 and authorization_errors_fatal:
return json_error(
_("Unable to access stream ({stream_name}).").format(
stream_name=unauthorized_streams[0].name,
)
)
# Newly created streams are also authorized for the creator
streams = authorized_streams + created_streams
if len(principals) > 0:
if realm.is_zephyr_mirror_realm and not all(stream.invite_only for stream in streams):
return json_error(
_("You can only invite other Zephyr mirroring users to private streams.")
)
if not user_profile.can_subscribe_other_users():
# Guest users case will not be handled here as it will
# be handled by the decorator above.
raise JsonableError(_("Insufficient permission"))
subscribers = {
principal_to_user_profile(user_profile, principal) for principal in principals
}
else:
subscribers = {user_profile}
(subscribed, already_subscribed) = bulk_add_subscriptions(
realm, streams, subscribers, acting_user=user_profile, color_map=color_map
)
# We can assume unique emails here for now, but we should eventually
# convert this function to be more id-centric.
email_to_user_profile: Dict[str, UserProfile] = {}
result: Dict[str, Any] = dict(
subscribed=defaultdict(list), already_subscribed=defaultdict(list)
)
for sub_info in subscribed:
subscriber = sub_info.user
stream = sub_info.stream
result["subscribed"][subscriber.email].append(stream.name)
email_to_user_profile[subscriber.email] = subscriber
for sub_info in already_subscribed:
subscriber = sub_info.user
stream = sub_info.stream
result["already_subscribed"][subscriber.email].append(stream.name)
result["subscribed"] = dict(result["subscribed"])
result["already_subscribed"] = dict(result["already_subscribed"])
send_messages_for_new_subscribers(
user_profile=user_profile,
subscribers=subscribers,
new_subscriptions=result["subscribed"],
email_to_user_profile=email_to_user_profile,
created_streams=created_streams,
announce=announce,
)
result["subscribed"] = dict(result["subscribed"])
result["already_subscribed"] = dict(result["already_subscribed"])
if not authorization_errors_fatal:
result["unauthorized"] = [s.name for s in unauthorized_streams]
return json_success(result)
def list_to_streams(
streams_raw: Iterable[StreamDict],
user_profile: UserProfile,
autocreate: bool = False,
admin_access_required: bool = False,
) -> Tuple[List[Stream], List[Stream]]:
"""Converts list of dicts to a list of Streams, validating input in the process
For each stream name, we validate it to ensure it meets our
requirements for a proper stream name using check_stream_name.
This function in autocreate mode should be atomic: either an exception will be raised
during a precheck, or all the streams specified will have been created if applicable.
@param streams_raw The list of stream dictionaries to process;
names should already be stripped of whitespace by the caller.
@param user_profile The user for whom we are retrieving the streams
@param autocreate Whether we should create streams if they don't already exist
"""
# Validate all streams, getting extant ones, then get-or-creating the rest.
stream_set = {stream_dict["name"] for stream_dict in streams_raw}
for stream_name in stream_set:
# Stream names should already have been stripped by the
# caller, but it makes sense to verify anyway.
assert stream_name == stream_name.strip()
check_stream_name(stream_name)
existing_streams: List[Stream] = []
missing_stream_dicts: List[StreamDict] = []
existing_stream_map = bulk_get_streams(user_profile.realm, stream_set)
if admin_access_required:
existing_recipient_ids = [
stream.recipient_id for stream in existing_stream_map.values()
]
subs = Subscription.objects.filter(
user_profile=user_profile,
recipient_id__in=existing_recipient_ids,
active=True)
sub_map = {sub.recipient_id: sub for sub in subs}
for stream in existing_stream_map.values():
sub = sub_map.get(stream.recipient_id, None)
check_stream_access_for_delete_or_update(user_profile, stream, sub)
message_retention_days_not_none = False
for stream_dict in streams_raw:
stream_name = stream_dict["name"]
stream = existing_stream_map.get(stream_name.lower())
if stream is None:
if stream_dict.get("message_retention_days", None) is not None:
message_retention_days_not_none = True
missing_stream_dicts.append(stream_dict)
else:
existing_streams.append(stream)
if len(missing_stream_dicts) == 0:
# This is the happy path for callers who expected all of these
# streams to exist already.
created_streams: List[Stream] = []
else:
# autocreate=True path starts here
if not user_profile.can_create_streams():
raise JsonableError(_("User cannot create streams."))
elif not autocreate:
raise JsonableError(
_("Stream(s) ({}) do not exist").format(
", ".join(stream_dict["name"]
for stream_dict in missing_stream_dicts), ))
elif message_retention_days_not_none:
if not user_profile.is_realm_owner:
raise JsonableError(
_("User cannot create stream with this settings."))
user_profile.realm.ensure_not_on_limited_plan()
# We already filtered out existing streams, so dup_streams
# will normally be an empty list below, but we protect against somebody
# else racing to create the same stream. (This is not an entirely
# paranoid approach, since often on Zulip two people will discuss
# creating a new stream, and both people eagerly do it.)
created_streams, dup_streams = create_streams_if_needed(
realm=user_profile.realm,
stream_dicts=missing_stream_dicts,
acting_user=user_profile)
existing_streams += dup_streams
return existing_streams, created_streams
def apply_event(state: Dict[str, Any], event: Dict[str, Any],
user_profile: UserProfile, client_gravatar: bool,
slim_presence: bool, include_subscribers: bool) -> None:
if event['type'] == "message":
state['max_message_id'] = max(state['max_message_id'],
event['message']['id'])
if 'raw_unread_msgs' in state:
apply_unread_message_event(
user_profile,
state['raw_unread_msgs'],
event['message'],
event['flags'],
)
if event['message']['type'] != "stream":
if 'raw_recent_private_conversations' in state:
# Handle maintaining the recent_private_conversations data structure.
conversations = state['raw_recent_private_conversations']
recipient_id = get_recent_conversations_recipient_id(
user_profile, event['message']['recipient_id'],
event['message']["sender_id"])
if recipient_id not in conversations:
conversations[recipient_id] = dict(user_ids=sorted([
user_dict['id']
for user_dict in event['message']['display_recipient']
if user_dict['id'] != user_profile.id
]))
conversations[recipient_id]['max_message_id'] = event[
'message']['id']
return
# Below, we handle maintaining first_message_id.
for sub_dict in state.get('subscriptions', []):
if event['message']['stream_id'] == sub_dict['stream_id']:
if sub_dict['first_message_id'] is None:
sub_dict['first_message_id'] = event['message']['id']
for stream_dict in state.get('streams', []):
if event['message']['stream_id'] == stream_dict['stream_id']:
if stream_dict['first_message_id'] is None:
stream_dict['first_message_id'] = event['message']['id']
elif event['type'] == "hotspots":
state['hotspots'] = event['hotspots']
elif event['type'] == "custom_profile_fields":
state['custom_profile_fields'] = event['fields']
elif event['type'] == "pointer":
state['pointer'] = max(state['pointer'], event['pointer'])
elif event['type'] == "realm_user":
person = event['person']
person_user_id = person['user_id']
if event['op'] == "add":
person = copy.deepcopy(person)
if client_gravatar:
if 'gravatar.com' in person['avatar_url']:
person['avatar_url'] = None
person['is_active'] = True
if not person['is_bot']:
person['profile_data'] = {}
state['raw_users'][person_user_id] = person
elif event['op'] == "remove":
state['raw_users'][person_user_id]['is_active'] = False
elif event['op'] == 'update':
is_me = (person_user_id == user_profile.id)
if is_me:
if ('avatar_url' in person and 'avatar_url' in state):
state['avatar_source'] = person['avatar_source']
state['avatar_url'] = person['avatar_url']
state['avatar_url_medium'] = person['avatar_url_medium']
for field in [
'is_admin', 'delivery_email', 'email', 'full_name'
]:
if field in person and field in state:
state[field] = person[field]
# In the unlikely event that the current user
# just changed to/from being an admin, we need
# to add/remove the data on all bots in the
# realm. This is ugly and probably better
# solved by removing the all-realm-bots data
# given to admin users from this flow.
if ('is_admin' in person and 'realm_bots' in state):
prev_state = state['raw_users'][user_profile.id]
was_admin = prev_state['is_admin']
now_admin = person['is_admin']
if was_admin and not now_admin:
state['realm_bots'] = []
if not was_admin and now_admin:
state['realm_bots'] = get_owned_bot_dicts(user_profile)
if client_gravatar and 'avatar_url' in person:
# Respect the client_gravatar setting in the `users` data.
if 'gravatar.com' in person['avatar_url']:
person['avatar_url'] = None
person['avatar_url_medium'] = None
if person_user_id in state['raw_users']:
p = state['raw_users'][person_user_id]
for field in p:
if field in person:
p[field] = person[field]
if 'custom_profile_field' in person:
custom_field_id = person['custom_profile_field']['id']
custom_field_new_value = person[
'custom_profile_field']['value']
if 'rendered_value' in person['custom_profile_field']:
p['profile_data'][custom_field_id] = {
'value':
custom_field_new_value,
'rendered_value':
person['custom_profile_field']
['rendered_value']
}
else:
p['profile_data'][custom_field_id] = {
'value': custom_field_new_value
}
elif event['type'] == 'realm_bot':
if event['op'] == 'add':
state['realm_bots'].append(event['bot'])
if event['op'] == 'remove':
email = event['bot']['email']
for bot in state['realm_bots']:
if bot['email'] == email:
bot['is_active'] = False
if event['op'] == 'delete':
state['realm_bots'] = [
item for item in state['realm_bots']
if item['email'] != event['bot']['email']
]
if event['op'] == 'update':
for bot in state['realm_bots']:
if bot['email'] == event['bot']['email']:
if 'owner_id' in event['bot']:
bot['owner'] = get_user_profile_by_id(
event['bot']['owner_id']).email
else:
bot.update(event['bot'])
elif event['type'] == 'stream':
if event['op'] == 'create':
for stream in event['streams']:
if not stream['invite_only']:
stream_data = copy.deepcopy(stream)
if include_subscribers:
stream_data['subscribers'] = []
stream_data['stream_weekly_traffic'] = None
stream_data['is_old_stream'] = False
stream_data[
'stream_post_policy'] = Stream.STREAM_POST_POLICY_EVERYONE
# Add stream to never_subscribed (if not invite_only)
state['never_subscribed'].append(stream_data)
state['streams'].append(stream)
state['streams'].sort(key=lambda elt: elt["name"])
if event['op'] == 'delete':
deleted_stream_ids = {
stream['stream_id']
for stream in event['streams']
}
state['streams'] = [
s for s in state['streams']
if s['stream_id'] not in deleted_stream_ids
]
state['never_subscribed'] = [
stream for stream in state['never_subscribed']
if stream['stream_id'] not in deleted_stream_ids
]
if event['op'] == 'update':
# For legacy reasons, we call stream data 'subscriptions' in
# the state var here, for the benefit of the JS code.
for obj in state['subscriptions']:
if obj['name'].lower() == event['name'].lower():
obj[event['property']] = event['value']
if event['property'] == "description":
obj['rendered_description'] = event[
'rendered_description']
# Also update the pure streams data
for stream in state['streams']:
if stream['name'].lower() == event['name'].lower():
prop = event['property']
if prop in stream:
stream[prop] = event['value']
if prop == 'description':
stream['rendered_description'] = event[
'rendered_description']
elif event['op'] == "occupy":
state['streams'] += event['streams']
elif event['op'] == "vacate":
stream_ids = [s["stream_id"] for s in event['streams']]
state['streams'] = [
s for s in state['streams'] if s["stream_id"] not in stream_ids
]
elif event['type'] == 'default_streams':
state['realm_default_streams'] = event['default_streams']
elif event['type'] == 'default_stream_groups':
state['realm_default_stream_groups'] = event['default_stream_groups']
elif event['type'] == 'realm':
if event['op'] == "update":
field = 'realm_' + event['property']
state[field] = event['value']
if event['property'] == 'plan_type':
# Then there are some extra fields that also need to be set.
state['plan_includes_wide_organization_logo'] = event[
'value'] != Realm.LIMITED
state['realm_upload_quota'] = event['extra_data'][
'upload_quota']
# Tricky interaction: Whether we can create streams can get changed here.
if (field in [
'realm_create_stream_policy',
'realm_waiting_period_threshold'
]) and 'can_create_streams' in state:
state['can_create_streams'] = user_profile.can_create_streams()
if (field in [
'realm_invite_to_stream_policy',
'realm_waiting_period_threshold'
]) and 'can_subscribe_other_users' in state:
state[
'can_subscribe_other_users'] = user_profile.can_subscribe_other_users(
)
elif event['op'] == "update_dict":
for key, value in event['data'].items():
state['realm_' + key] = value
# It's a bit messy, but this is where we need to
# update the state for whether password authentication
# is enabled on this server.
if key == 'authentication_methods':
state['realm_password_auth_enabled'] = (value['Email']
or value['LDAP'])
state['realm_email_auth_enabled'] = value['Email']
elif event['type'] == "subscription":
if not include_subscribers and event['op'] in [
'peer_add', 'peer_remove'
]:
return
if event['op'] in ["add"]:
if not include_subscribers:
# Avoid letting 'subscribers' entries end up in the list
for i, sub in enumerate(event['subscriptions']):
event['subscriptions'][i] = copy.deepcopy(
event['subscriptions'][i])
del event['subscriptions'][i]['subscribers']
def name(sub: Dict[str, Any]) -> str:
return sub['name'].lower()
if event['op'] == "add":
added_names = set(map(name, event["subscriptions"]))
was_added = lambda s: name(s) in added_names
# add the new subscriptions
state['subscriptions'] += event['subscriptions']
# remove them from unsubscribed if they had been there
state['unsubscribed'] = [
s for s in state['unsubscribed'] if not was_added(s)
]
# remove them from never_subscribed if they had been there
state['never_subscribed'] = [
s for s in state['never_subscribed'] if not was_added(s)
]
elif event['op'] == "remove":
removed_names = set(map(name, event["subscriptions"]))
was_removed = lambda s: name(s) in removed_names
# Find the subs we are affecting.
removed_subs = list(filter(was_removed, state['subscriptions']))
# Remove our user from the subscribers of the removed subscriptions.
if include_subscribers:
for sub in removed_subs:
sub['subscribers'].remove(user_profile.id)
# We must effectively copy the removed subscriptions from subscriptions to
# unsubscribe, since we only have the name in our data structure.
state['unsubscribed'] += removed_subs
# Now filter out the removed subscriptions from subscriptions.
state['subscriptions'] = [
s for s in state['subscriptions'] if not was_removed(s)
]
elif event['op'] == 'update':
for sub in state['subscriptions']:
if sub['name'].lower() == event['name'].lower():
sub[event['property']] = event['value']
elif event['op'] == 'peer_add':
user_id = event['user_id']
for sub in state['subscriptions']:
if (sub['name'] in event['subscriptions']
and user_id not in sub['subscribers']):
sub['subscribers'].append(user_id)
for sub in state['never_subscribed']:
if (sub['name'] in event['subscriptions']
and user_id not in sub['subscribers']):
sub['subscribers'].append(user_id)
elif event['op'] == 'peer_remove':
user_id = event['user_id']
for sub in state['subscriptions']:
if (sub['name'] in event['subscriptions']
and user_id in sub['subscribers']):
sub['subscribers'].remove(user_id)
elif event['type'] == "presence":
if slim_presence:
user_key = str(event['user_id'])
else:
user_key = event['email']
state['presences'][user_key] = get_presence_for_user(
event['user_id'], slim_presence)[user_key]
elif event['type'] == "update_message":
# We don't return messages in /register, so we don't need to
# do anything for content updates, but we may need to update
# the unread_msgs data if the topic of an unread message changed.
if TOPIC_NAME in event:
stream_dict = state['raw_unread_msgs']['stream_dict']
topic = event[TOPIC_NAME]
for message_id in event['message_ids']:
if message_id in stream_dict:
stream_dict[message_id]['topic'] = topic
elif event['type'] == "delete_message":
max_message = Message.objects.filter(
usermessage__user_profile=user_profile).order_by('-id').first()
if max_message:
state['max_message_id'] = max_message.id
else:
state['max_message_id'] = -1
if 'raw_unread_msgs' in state:
remove_id = event['message_id']
remove_message_id_from_unread_mgs(state['raw_unread_msgs'],
remove_id)
# The remainder of this block is about maintaining recent_private_conversations
if 'raw_recent_private_conversations' not in state or event[
'message_type'] != 'private':
return
recipient_id = get_recent_conversations_recipient_id(
user_profile, event['recipient_id'], event['sender_id'])
# Ideally, we'd have test coverage for these two blocks. To
# do that, we'll need a test where we delete not-the-latest
# messages or delete a private message not in
# recent_private_conversations.
if recipient_id not in state[
'raw_recent_private_conversations']: # nocoverage
return
old_max_message_id = state['raw_recent_private_conversations'][
recipient_id]['max_message_id']
if old_max_message_id != event['message_id']: # nocoverage
return
# OK, we just deleted what had been the max_message_id for
# this recent conversation; we need to recompute that value
# from scratch. Definitely don't need to re-query everything,
# but this case is likely rare enough that it's reasonable to do so.
state['raw_recent_private_conversations'] = \
get_recent_private_conversations(user_profile)
elif event['type'] == "reaction":
# The client will get the message with the reactions directly
pass
elif event['type'] == "submessage":
# The client will get submessages with their messages
pass
elif event['type'] == 'typing':
# Typing notification events are transient and thus ignored
pass
elif event['type'] == "attachment":
# Attachment events are just for updating the "uploads" UI;
# they are not sent directly.
pass
elif event['type'] == "update_message_flags":
# We don't return messages in `/register`, so most flags we
# can ignore, but we do need to update the unread_msgs data if
# unread state is changed.
if 'raw_unread_msgs' in state and event['flag'] == 'read' and event[
'operation'] == 'add':
for remove_id in event['messages']:
remove_message_id_from_unread_mgs(state['raw_unread_msgs'],
remove_id)
if event['flag'] == 'starred' and event['operation'] == 'add':
state['starred_messages'] += event['messages']
if event['flag'] == 'starred' and event['operation'] == 'remove':
state['starred_messages'] = [
message for message in state['starred_messages']
if not (message in event['messages'])
]
elif event['type'] == "realm_domains":
if event['op'] == 'add':
state['realm_domains'].append(event['realm_domain'])
elif event['op'] == 'change':
for realm_domain in state['realm_domains']:
if realm_domain['domain'] == event['realm_domain']['domain']:
realm_domain['allow_subdomains'] = event['realm_domain'][
'allow_subdomains']
elif event['op'] == 'remove':
state['realm_domains'] = [
realm_domain for realm_domain in state['realm_domains']
if realm_domain['domain'] != event['domain']
]
elif event['type'] == "realm_emoji":
state['realm_emoji'] = event['realm_emoji']
elif event['type'] == 'realm_export':
# These realm export events are only available to
# administrators, and aren't included in page_params.
pass
elif event['type'] == "alert_words":
state['alert_words'] = event['alert_words']
elif event['type'] == "muted_topics":
state['muted_topics'] = event["muted_topics"]
elif event['type'] == "realm_filters":
state['realm_filters'] = event["realm_filters"]
elif event['type'] == "update_display_settings":
assert event['setting_name'] in UserProfile.property_types
state[event['setting_name']] = event['setting']
elif event['type'] == "update_global_notifications":
assert event[
'notification_name'] in UserProfile.notification_setting_types
state[event['notification_name']] = event['setting']
elif event['type'] == "invites_changed":
pass
elif event['type'] == "user_group":
if event['op'] == 'add':
state['realm_user_groups'].append(event['group'])
state['realm_user_groups'].sort(key=lambda group: group['id'])
elif event['op'] == 'update':
for user_group in state['realm_user_groups']:
if user_group['id'] == event['group_id']:
user_group.update(event['data'])
elif event['op'] == 'add_members':
for user_group in state['realm_user_groups']:
if user_group['id'] == event['group_id']:
user_group['members'].extend(event['user_ids'])
user_group['members'].sort()
elif event['op'] == 'remove_members':
for user_group in state['realm_user_groups']:
if user_group['id'] == event['group_id']:
members = set(user_group['members'])
user_group['members'] = list(members -
set(event['user_ids']))
user_group['members'].sort()
elif event['op'] == 'remove':
state['realm_user_groups'] = [
ug for ug in state['realm_user_groups']
if ug['id'] != event['group_id']
]
elif event['type'] == 'user_status':
user_id = event['user_id']
user_status = state['user_status']
away = event.get('away')
status_text = event.get('status_text')
if user_id not in user_status:
user_status[user_id] = dict()
if away is not None:
if away:
user_status[user_id]['away'] = True
else:
user_status[user_id].pop('away', None)
if status_text is not None:
if status_text == '':
user_status[user_id].pop('status_text', None)
else:
user_status[user_id]['status_text'] = status_text
if not user_status[user_id]:
user_status.pop(user_id, None)
state['user_status'] = user_status
else:
raise AssertionError("Unexpected event type %s" % (event['type'], ))
def users_to_zerver_userprofile(
slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int,
timestamp: Any, domain_name: str
) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT], AddedUsersT,
List[ZerverFieldsT], List[ZerverFieldsT]]:
"""
Returns:
1. zerver_userprofile, which is a list of user profile
2. avatar_list, which is list to map avatars to zulip avatard records.json
3. added_users, which is a dictionary to map from slack user id to zulip
user id
4. zerver_customprofilefield, which is a list of all custom profile fields
5. zerver_customprofilefield_values, which is a list of user profile fields
"""
logging.info('######### IMPORTING USERS STARTED #########\n')
zerver_userprofile = []
zerver_customprofilefield = [] # type: List[ZerverFieldsT]
zerver_customprofilefield_values = [] # type: List[ZerverFieldsT]
avatar_list = [] # type: List[ZerverFieldsT]
added_users = {}
# The user data we get from the slack api does not contain custom profile data
# Hence we get it from the slack zip file
slack_data_file_user_list = get_data_file(slack_data_dir + '/users.json')
# To map user id with the custom profile fields of the corresponding user
slack_user_custom_field_map = {} # type: ZerverFieldsT
# To store custom fields corresponding to their ids
custom_field_map = {} # type: ZerverFieldsT
for user in slack_data_file_user_list:
process_slack_custom_fields(user, slack_user_custom_field_map)
# We have only one primary owner in slack, see link
# https://get.slack.help/hc/en-us/articles/201912948-Owners-and-Administrators
# This is to import the primary owner first from all the users
user_id_count = custom_field_id_count = customprofilefield_id = 0
primary_owner_id = user_id_count
user_id_count += 1
for user in users:
slack_user_id = user['id']
if user.get('is_primary_owner', False):
user_id = primary_owner_id
else:
user_id = user_id_count
# email
email = get_user_email(user, domain_name)
# avatar
# ref: https://chat.zulip.org/help/change-your-avatar
avatar_url = build_avatar_url(slack_user_id, user['team_id'],
user['profile']['avatar_hash'])
build_avatar(user_id, realm_id, email, avatar_url, timestamp,
avatar_list)
# check if user is the admin
realm_admin = get_admin(user)
# timezone
timezone = get_user_timezone(user)
# Check for custom profile fields
if slack_user_id in slack_user_custom_field_map:
# For processing the fields
custom_field_map, customprofilefield_id = build_customprofile_field(
zerver_customprofilefield,
slack_user_custom_field_map[slack_user_id],
customprofilefield_id, realm_id, custom_field_map)
# Store the custom field values for the corresponding user
custom_field_id_count = build_customprofilefields_values(
custom_field_map, slack_user_custom_field_map[slack_user_id],
user_id, custom_field_id_count,
zerver_customprofilefield_values)
userprofile = UserProfile(
full_name=get_user_full_name(user),
short_name=user['name'],
is_active=not user['deleted'],
id=user_id,
email=email,
delivery_email=email,
avatar_source='U',
is_bot=user.get('is_bot', False),
pointer=-1,
is_realm_admin=realm_admin,
bot_type=1 if user.get('is_bot', False) else None,
date_joined=timestamp,
timezone=timezone,
last_login=timestamp)
userprofile_dict = model_to_dict(userprofile)
# Set realm id separately as the corresponding realm is not yet a Realm model instance
userprofile_dict['realm'] = realm_id
zerver_userprofile.append(userprofile_dict)
added_users[slack_user_id] = user_id
if not user.get('is_primary_owner', False):
user_id_count += 1
logging.info(u"{} -> {}".format(user['name'],
userprofile_dict['email']))
process_customprofilefields(zerver_customprofilefield,
zerver_customprofilefield_values)
logging.info('######### IMPORTING USERS FINISHED #########\n')
return zerver_userprofile, avatar_list, added_users, zerver_customprofilefield, \
zerver_customprofilefield_values
def users_to_zerver_userprofile(
slack_data_dir: str, users: List[ZerverFieldsT], realm_id: int,
timestamp: Any, domain_name: str
) -> Tuple[List[ZerverFieldsT], List[ZerverFieldsT], SlackToZulipUserIDT,
List[ZerverFieldsT], List[ZerverFieldsT]]:
"""
Returns:
1. zerver_userprofile, which is a list of user profile
2. avatar_list, which is list to map avatars to zulip avatard records.json
3. slack_user_id_to_zulip_user_id, which is a dictionary to map from slack user id to zulip
user id
4. zerver_customprofilefield, which is a list of all custom profile fields
5. zerver_customprofilefield_values, which is a list of user profile fields
"""
logging.info('######### IMPORTING USERS STARTED #########\n')
zerver_userprofile = []
zerver_customprofilefield = [] # type: List[ZerverFieldsT]
zerver_customprofilefield_values = [] # type: List[ZerverFieldsT]
avatar_list = [] # type: List[ZerverFieldsT]
slack_user_id_to_zulip_user_id = {}
# The user data we get from the slack api does not contain custom profile data
# Hence we get it from the slack zip file
slack_data_file_user_list = get_data_file(slack_data_dir + '/users.json')
slack_user_id_to_custom_profile_fields = {} # type: ZerverFieldsT
slack_custom_field_name_to_zulip_custom_field_id = {
} # type: ZerverFieldsT
for user in slack_data_file_user_list:
process_slack_custom_fields(user,
slack_user_id_to_custom_profile_fields)
# We have only one primary owner in slack, see link
# https://get.slack.help/hc/en-us/articles/201912948-Owners-and-Administrators
# This is to import the primary owner first from all the users
user_id_count = custom_profile_field_value_id_count = custom_profile_field_id_count = 0
primary_owner_id = user_id_count
user_id_count += 1
for user in users:
slack_user_id = user['id']
if user.get('is_primary_owner', False):
user_id = primary_owner_id
else:
user_id = user_id_count
email = get_user_email(user, domain_name)
# ref: https://chat.zulip.org/help/set-your-profile-picture
avatar_url = build_avatar_url(slack_user_id, user['team_id'],
user['profile']['avatar_hash'])
build_avatar(user_id, realm_id, email, avatar_url, timestamp,
avatar_list)
role = UserProfile.ROLE_MEMBER
if get_admin(user):
role = UserProfile.ROLE_REALM_ADMINISTRATOR
if get_guest(user):
role = UserProfile.ROLE_GUEST
timezone = get_user_timezone(user)
if slack_user_id in slack_user_id_to_custom_profile_fields:
slack_custom_field_name_to_zulip_custom_field_id, custom_profile_field_id_count = \
build_customprofile_field(zerver_customprofilefield,
slack_user_id_to_custom_profile_fields[slack_user_id],
custom_profile_field_id_count, realm_id,
slack_custom_field_name_to_zulip_custom_field_id)
custom_profile_field_value_id_count = build_customprofilefields_values(
slack_custom_field_name_to_zulip_custom_field_id,
slack_user_id_to_custom_profile_fields[slack_user_id], user_id,
custom_profile_field_value_id_count,
zerver_customprofilefield_values)
userprofile = UserProfile(
full_name=get_user_full_name(user),
short_name=user['name'],
is_active=not user.get('deleted', False)
and not user["is_mirror_dummy"],
is_mirror_dummy=user["is_mirror_dummy"],
id=user_id,
email=email,
delivery_email=email,
avatar_source='U',
is_bot=user.get('is_bot', False),
pointer=-1,
role=role,
bot_type=1 if user.get('is_bot', False) else None,
date_joined=timestamp,
timezone=timezone,
last_login=timestamp)
userprofile_dict = model_to_dict(userprofile)
# Set realm id separately as the corresponding realm is not yet a Realm model instance
userprofile_dict['realm'] = realm_id
zerver_userprofile.append(userprofile_dict)
slack_user_id_to_zulip_user_id[slack_user_id] = user_id
if not user.get('is_primary_owner', False):
user_id_count += 1
logging.info(u"{} -> {}".format(user['name'],
userprofile_dict['email']))
process_customprofilefields(zerver_customprofilefield,
zerver_customprofilefield_values)
logging.info('######### IMPORTING USERS FINISHED #########\n')
return zerver_userprofile, avatar_list, slack_user_id_to_zulip_user_id, zerver_customprofilefield, \
zerver_customprofilefield_values
def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile,
missed_messages: List[Dict[str, Any]],
message_count: int) -> None:
"""
Send a reminder email to a user if she's missed some PMs by being offline.
The email will have its reply to address set to a limited used email
address that will send a zulip message to the correct recipient. This
allows the user to respond to missed PMs, huddles, and @-mentions directly
from the email.
`user_profile` is the user to send the reminder to
`missed_messages` is a list of dictionaries to Message objects and other data
for a group of messages that share a recipient (and topic)
"""
from zerver.context_processors import common_context
# Disabled missedmessage emails internally
if not user_profile.enable_offline_email_notifications:
return
recipients = set((msg['message'].recipient_id, msg['message'].topic_name()) for msg in missed_messages)
if len(recipients) != 1:
raise ValueError(
'All missed_messages must have the same recipient and topic %r' %
recipients
)
unsubscribe_link = one_click_unsubscribe_link(user_profile, "missed_messages")
context = common_context(user_profile)
context.update({
'name': user_profile.full_name,
'message_count': message_count,
'unsubscribe_link': unsubscribe_link,
'realm_name_in_notifications': user_profile.realm_name_in_notifications,
'show_message_content': user_profile.message_content_in_email_notifications,
})
triggers = list(message['trigger'] for message in missed_messages)
unique_triggers = set(triggers)
context.update({
'mention': 'mentioned' in unique_triggers,
'mention_count': triggers.count('mentioned'),
})
# If this setting (email mirroring integration) is enabled, only then
# can users reply to email to send message to Zulip. Thus, one must
# ensure to display warning in the template.
if settings.EMAIL_GATEWAY_PATTERN:
context.update({
'reply_warning': False,
'reply_to_zulip': True,
})
else:
context.update({
'reply_warning': True,
'reply_to_zulip': False,
})
from zerver.lib.email_mirror import create_missed_message_address
reply_to_address = create_missed_message_address(user_profile, missed_messages[0]['message'])
if reply_to_address == FromAddress.NOREPLY:
reply_to_name = None
else:
reply_to_name = "Zulip"
senders = list(set(m['message'].sender for m in missed_messages))
if (missed_messages[0]['message'].recipient.type == Recipient.HUDDLE):
display_recipient = get_display_recipient(missed_messages[0]['message'].recipient)
# Make sure that this is a list of strings, not a string.
assert not isinstance(display_recipient, str)
other_recipients = [r['full_name'] for r in display_recipient
if r['id'] != user_profile.id]
context.update({'group_pm': True})
if len(other_recipients) == 2:
huddle_display_name = "%s" % (" and ".join(other_recipients))
context.update({'huddle_display_name': huddle_display_name})
elif len(other_recipients) == 3:
huddle_display_name = "%s, %s, and %s" % (
other_recipients[0], other_recipients[1], other_recipients[2])
context.update({'huddle_display_name': huddle_display_name})
else:
huddle_display_name = "%s, and %s others" % (
', '.join(other_recipients[:2]), len(other_recipients) - 2)
context.update({'huddle_display_name': huddle_display_name})
elif (missed_messages[0]['message'].recipient.type == Recipient.PERSONAL):
context.update({'private_message': True})
elif context['mention']:
# Keep only the senders who actually mentioned the user
senders = list(set(m['message'].sender for m in missed_messages
if m['trigger'] == 'mentioned'))
# TODO: When we add wildcard mentions that send emails, we
# should make sure the right logic applies here.
elif ('stream_email_notify' in unique_triggers):
context.update({'stream_email_notify': True})
else:
raise AssertionError("Invalid messages!")
# If message content is disabled, then flush all information we pass to email.
if not user_profile.message_content_in_email_notifications:
context.update({
'reply_to_zulip': False,
'messages': [],
'sender_str': "",
'realm_str': user_profile.realm.name,
'huddle_display_name': "",
})
else:
context.update({
'messages': build_message_list(user_profile, list(m['message'] for m in missed_messages)),
'sender_str': ", ".join(sender.full_name for sender in senders),
'realm_str': user_profile.realm.name,
})
from_name = "Zulip missed messages" # type: str
from_address = FromAddress.NOREPLY
if len(senders) == 1 and settings.SEND_MISSED_MESSAGE_EMAILS_AS_USER:
# If this setting is enabled, you can reply to the Zulip
# missed message emails directly back to the original sender.
# However, one must ensure the Zulip server is in the SPF
# record for the domain, or there will be spam/deliverability
# problems.
sender = senders[0]
from_name, from_address = (sender.full_name, sender.email)
context.update({
'reply_warning': False,
'reply_to_zulip': False,
})
email_dict = {
'template_prefix': 'zerver/emails/missed_message',
'to_user_id': user_profile.id,
'from_name': from_name,
'from_address': from_address,
'reply_to_email': formataddr((reply_to_name, reply_to_address)),
'context': context}
queue_json_publish("email_senders", email_dict)
user_profile.last_reminder = timezone_now()
user_profile.save(update_fields=['last_reminder'])
def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile,
missed_messages: List[Dict[
str, Any]],
message_count: int) -> None:
"""
Send a reminder email to a user if she's missed some PMs by being offline.
The email will have its reply to address set to a limited used email
address that will send a Zulip message to the correct recipient. This
allows the user to respond to missed PMs, huddles, and @-mentions directly
from the email.
`user_profile` is the user to send the reminder to
`missed_messages` is a list of dictionaries to Message objects and other data
for a group of messages that share a recipient (and topic)
"""
from zerver.context_processors import common_context
# Disabled missedmessage emails internally
if not user_profile.enable_offline_email_notifications:
return
recipients = {(msg["message"].recipient_id, msg["message"].topic_name())
for msg in missed_messages}
if len(recipients) != 1:
raise ValueError(
f"All missed_messages must have the same recipient and topic {recipients!r}",
)
# This link is no longer a part of the email, but keeping the code in case
# we find a clean way to add it back in the future
unsubscribe_link = one_click_unsubscribe_link(user_profile,
"missed_messages")
context = common_context(user_profile)
context.update(
name=user_profile.full_name,
message_count=message_count,
unsubscribe_link=unsubscribe_link,
realm_name_in_notifications=user_profile.realm_name_in_notifications,
)
triggers = [message["trigger"] for message in missed_messages]
unique_triggers = set(triggers)
context.update(
mention="mentioned" in unique_triggers
or "wildcard_mentioned" in unique_triggers,
stream_email_notify="stream_email_notify" in unique_triggers,
mention_count=triggers.count("mentioned") +
triggers.count("wildcard_mentioned"),
)
# If this setting (email mirroring integration) is enabled, only then
# can users reply to email to send message to Zulip. Thus, one must
# ensure to display warning in the template.
if settings.EMAIL_GATEWAY_PATTERN:
context.update(reply_to_zulip=True, )
else:
context.update(reply_to_zulip=False, )
from zerver.lib.email_mirror import create_missed_message_address
reply_to_address = create_missed_message_address(
user_profile, missed_messages[0]["message"])
if reply_to_address == FromAddress.NOREPLY:
reply_to_name = ""
else:
reply_to_name = "Zulip"
narrow_url = get_narrow_url(user_profile, missed_messages[0]["message"])
context.update(narrow_url=narrow_url, )
senders = list({m["message"].sender for m in missed_messages})
if missed_messages[0]["message"].recipient.type == Recipient.HUDDLE:
display_recipient = get_display_recipient(
missed_messages[0]["message"].recipient)
# Make sure that this is a list of strings, not a string.
assert not isinstance(display_recipient, str)
other_recipients = [
r["full_name"] for r in display_recipient
if r["id"] != user_profile.id
]
context.update(group_pm=True)
if len(other_recipients) == 2:
huddle_display_name = " and ".join(other_recipients)
context.update(huddle_display_name=huddle_display_name)
elif len(other_recipients) == 3:
huddle_display_name = (
f"{other_recipients[0]}, {other_recipients[1]}, and {other_recipients[2]}"
)
context.update(huddle_display_name=huddle_display_name)
else:
huddle_display_name = "{}, and {} others".format(
", ".join(other_recipients[:2]),
len(other_recipients) - 2)
context.update(huddle_display_name=huddle_display_name)
elif missed_messages[0]["message"].recipient.type == Recipient.PERSONAL:
context.update(private_message=True)
elif context["mention"] or context["stream_email_notify"]:
# Keep only the senders who actually mentioned the user
if context["mention"]:
senders = list({
m["message"].sender
for m in missed_messages if m["trigger"] == "mentioned"
or m["trigger"] == "wildcard_mentioned"
})
message = missed_messages[0]["message"]
stream = Stream.objects.only("id",
"name").get(id=message.recipient.type_id)
stream_header = f"{stream.name} > {message.topic_name()}"
context.update(stream_header=stream_header, )
else:
raise AssertionError("Invalid messages!")
# If message content is disabled, then flush all information we pass to email.
if not message_content_allowed_in_missedmessage_emails(user_profile):
realm = user_profile.realm
context.update(
reply_to_zulip=False,
messages=[],
sender_str="",
realm_str=realm.name,
huddle_display_name="",
show_message_content=False,
message_content_disabled_by_user=not user_profile.
message_content_in_email_notifications,
message_content_disabled_by_realm=not realm.
message_content_allowed_in_email_notifications,
)
else:
context.update(
messages=build_message_list(
user=user_profile,
messages=[m["message"] for m in missed_messages],
stream_map={},
),
sender_str=", ".join(sender.full_name for sender in senders),
realm_str=user_profile.realm.name,
show_message_content=True,
)
with override_language(user_profile.default_language):
from_name: str = _("Zulip notifications")
from_address = FromAddress.NOREPLY
if len(senders) == 1 and settings.SEND_MISSED_MESSAGE_EMAILS_AS_USER:
# If this setting is enabled, you can reply to the Zulip
# message notification emails directly back to the original sender.
# However, one must ensure the Zulip server is in the SPF
# record for the domain, or there will be spam/deliverability
# problems.
#
# Also, this setting is not really compatible with
# EMAIL_ADDRESS_VISIBILITY_ADMINS.
sender = senders[0]
from_name, from_address = (sender.full_name, sender.email)
context.update(reply_to_zulip=False, )
email_dict = {
"template_prefix":
"zerver/emails/missed_message",
"to_user_ids": [user_profile.id],
"from_name":
from_name,
"from_address":
from_address,
"reply_to_email":
str(Address(display_name=reply_to_name, addr_spec=reply_to_address)),
"context":
context,
}
queue_json_publish("email_senders", email_dict)
user_profile.last_reminder = timezone_now()
user_profile.save(update_fields=["last_reminder"])
def add_subscriptions_backend(
request: HttpRequest,
user_profile: UserProfile,
streams_raw: Iterable[Dict[str,
str]] = REQ("subscriptions",
validator=add_subscriptions_schema),
invite_only: bool = REQ(validator=check_bool, default=False),
stream_post_policy: int = REQ(validator=check_int_in(
Stream.STREAM_POST_POLICY_TYPES),
default=Stream.STREAM_POST_POLICY_EVERYONE),
history_public_to_subscribers: Optional[bool] = REQ(validator=check_bool,
default=None),
message_retention_days: Union[str,
int] = REQ(validator=check_string_or_int,
default=RETENTION_DEFAULT),
announce: bool = REQ(validator=check_bool, default=False),
principals: Union[Sequence[str], Sequence[int]] = REQ(
validator=check_principals,
default=EMPTY_PRINCIPALS,
),
authorization_errors_fatal: bool = REQ(validator=check_bool, default=True),
) -> HttpResponse:
stream_dicts = []
color_map = {}
for stream_dict in streams_raw:
# 'color' field is optional
# check for its presence in the streams_raw first
if 'color' in stream_dict:
color_map[stream_dict['name']] = stream_dict['color']
if 'description' in stream_dict:
# We don't allow newline characters in stream descriptions.
stream_dict['description'] = stream_dict['description'].replace(
"\n", " ")
stream_dict_copy: Dict[str, Any] = {}
for field in stream_dict:
stream_dict_copy[field] = stream_dict[field]
# Strip the stream name here.
stream_dict_copy['name'] = stream_dict_copy['name'].strip()
stream_dict_copy["invite_only"] = invite_only
stream_dict_copy["stream_post_policy"] = stream_post_policy
stream_dict_copy[
"history_public_to_subscribers"] = history_public_to_subscribers
stream_dict_copy[
"message_retention_days"] = parse_message_retention_days(
message_retention_days,
Stream.MESSAGE_RETENTION_SPECIAL_VALUES_MAP)
stream_dicts.append(stream_dict_copy)
# Validation of the streams arguments, including enforcement of
# can_create_streams policy and check_stream_name policy is inside
# list_to_streams.
existing_streams, created_streams = \
list_to_streams(stream_dicts, user_profile, autocreate=True)
authorized_streams, unauthorized_streams = \
filter_stream_authorization(user_profile, existing_streams)
if len(unauthorized_streams) > 0 and authorization_errors_fatal:
return json_error(
_("Unable to access stream ({stream_name}).").format(
stream_name=unauthorized_streams[0].name, ))
# Newly created streams are also authorized for the creator
streams = authorized_streams + created_streams
if len(principals) > 0:
if user_profile.realm.is_zephyr_mirror_realm and not all(
stream.invite_only for stream in streams):
return json_error(
_("You can only invite other Zephyr mirroring users to private streams."
))
if not user_profile.can_subscribe_other_users():
if user_profile.realm.invite_to_stream_policy == Realm.POLICY_ADMINS_ONLY:
return json_error(
_("Only administrators can modify other users' subscriptions."
))
# Realm.POLICY_MEMBERS_ONLY only fails if the
# user is a guest, which happens in the decorator above.
assert user_profile.realm.invite_to_stream_policy == \
Realm.POLICY_FULL_MEMBERS_ONLY
return json_error(
_("Your account is too new to modify other users' subscriptions."
))
subscribers = {
principal_to_user_profile(user_profile, principal)
for principal in principals
}
else:
subscribers = {user_profile}
(subscribed,
already_subscribed) = bulk_add_subscriptions(streams,
subscribers,
acting_user=user_profile,
color_map=color_map)
# We can assume unique emails here for now, but we should eventually
# convert this function to be more id-centric.
email_to_user_profile: Dict[str, UserProfile] = dict()
result: Dict[str, Any] = dict(subscribed=defaultdict(list),
already_subscribed=defaultdict(list))
for (subscriber, stream) in subscribed:
result["subscribed"][subscriber.email].append(stream.name)
email_to_user_profile[subscriber.email] = subscriber
for (subscriber, stream) in already_subscribed:
result["already_subscribed"][subscriber.email].append(stream.name)
bots = {subscriber.email: subscriber.is_bot for subscriber in subscribers}
newly_created_stream_names = {s.name for s in created_streams}
# Inform the user if someone else subscribed them to stuff,
# or if a new stream was created with the "announce" option.
notifications = []
if len(principals) > 0 and result["subscribed"]:
for email, subscribed_stream_names in result["subscribed"].items():
if email == user_profile.email:
# Don't send a Zulip if you invited yourself.
continue
if bots[email]:
# Don't send invitation Zulips to bots
continue
# For each user, we notify them about newly subscribed streams, except for
# streams that were newly created.
notify_stream_names = set(
subscribed_stream_names) - newly_created_stream_names
if not notify_stream_names:
continue
sender = get_system_bot(settings.NOTIFICATION_BOT)
recipient_user = email_to_user_profile[email]
msg = you_were_just_subscribed_message(
acting_user=user_profile,
recipient_user=recipient_user,
stream_names=notify_stream_names,
)
notifications.append(
internal_prep_private_message(realm=user_profile.realm,
sender=sender,
recipient_user=recipient_user,
content=msg))
if announce and len(created_streams) > 0:
notifications_stream = user_profile.realm.get_notifications_stream()
if notifications_stream is not None:
with override_language(
notifications_stream.realm.default_language):
if len(created_streams) > 1:
content = _(
"{user_name} created the following streams: {stream_str}."
)
else:
content = _(
"{user_name} created a new stream {stream_str}.")
topic = _('new streams')
content = content.format(
user_name=f"@_**{user_profile.full_name}|{user_profile.id}**",
stream_str=", ".join(f'#**{s.name}**'
for s in created_streams))
sender = get_system_bot(settings.NOTIFICATION_BOT)
notifications.append(
internal_prep_stream_message(
realm=user_profile.realm,
sender=sender,
stream=notifications_stream,
topic=topic,
content=content,
), )
if not user_profile.realm.is_zephyr_mirror_realm and len(
created_streams) > 0:
sender = get_system_bot(settings.NOTIFICATION_BOT)
for stream in created_streams:
with override_language(stream.realm.default_language):
notifications.append(
internal_prep_stream_message(
realm=user_profile.realm,
sender=sender,
stream=stream,
topic=Realm.STREAM_EVENTS_NOTIFICATION_TOPIC,
content=_('Stream created by {user_name}.').format(
user_name=
f"@_**{user_profile.full_name}|{user_profile.id}**",
),
), )
if len(notifications) > 0:
do_send_messages(notifications, mark_as_read=[user_profile.id])
result["subscribed"] = dict(result["subscribed"])
result["already_subscribed"] = dict(result["already_subscribed"])
if not authorization_errors_fatal:
result["unauthorized"] = [s.name for s in unauthorized_streams]
return json_success(result)
def apply_event(state: Dict[str, Any], event: Dict[str, Any],
user_profile: UserProfile, client_gravatar: bool,
include_subscribers: bool) -> None:
if event['type'] == "message":
state['max_message_id'] = max(state['max_message_id'],
event['message']['id'])
if 'raw_unread_msgs' in state:
apply_unread_message_event(
user_profile,
state['raw_unread_msgs'],
event['message'],
event['flags'],
)
elif event['type'] == "hotspots":
state['hotspots'] = event['hotspots']
elif event['type'] == "custom_profile_fields":
state['custom_profile_fields'] = event['fields']
elif event['type'] == "pointer":
state['pointer'] = max(state['pointer'], event['pointer'])
elif event['type'] == "realm_user":
person = event['person']
person_user_id = person['user_id']
if event['op'] == "add":
person = copy.deepcopy(person)
if client_gravatar:
if 'gravatar.com' in person['avatar_url']:
person['avatar_url'] = None
person['is_active'] = True
if not person['is_bot']:
person['profile_data'] = {}
state['raw_users'][person_user_id] = person
elif event['op'] == "remove":
state['raw_users'][person_user_id]['is_active'] = False
elif event['op'] == 'update':
is_me = (person_user_id == user_profile.id)
if is_me:
if ('avatar_url' in person and 'avatar_url' in state):
state['avatar_source'] = person['avatar_source']
state['avatar_url'] = person['avatar_url']
state['avatar_url_medium'] = person['avatar_url_medium']
for field in ['is_admin', 'email', 'full_name']:
if field in person and field in state:
state[field] = person[field]
# In the unlikely event that the current user
# just changed to/from being an admin, we need
# to add/remove the data on all bots in the
# realm. This is ugly and probably better
# solved by removing the all-realm-bots data
# given to admin users from this flow.
if ('is_admin' in person and 'realm_bots' in state):
prev_state = state['raw_users'][user_profile.id]
was_admin = prev_state['is_admin']
now_admin = person['is_admin']
if was_admin and not now_admin:
state['realm_bots'] = []
if not was_admin and now_admin:
state['realm_bots'] = get_owned_bot_dicts(user_profile)
if client_gravatar and 'avatar_url' in person:
# Respect the client_gravatar setting in the `users` data.
if 'gravatar.com' in person['avatar_url']:
person['avatar_url'] = None
person['avatar_url_medium'] = None
if person_user_id in state['raw_users']:
p = state['raw_users'][person_user_id]
for field in p:
if field in person:
p[field] = person[field]
elif event['type'] == 'realm_bot':
if event['op'] == 'add':
state['realm_bots'].append(event['bot'])
if event['op'] == 'remove':
email = event['bot']['email']
for bot in state['realm_bots']:
if bot['email'] == email:
bot['is_active'] = False
if event['op'] == 'delete':
state['realm_bots'] = [
item for item in state['realm_bots']
if item['email'] != event['bot']['email']
]
if event['op'] == 'update':
for bot in state['realm_bots']:
if bot['email'] == event['bot']['email']:
if 'owner_id' in event['bot']:
bot['owner'] = get_user_profile_by_id(
event['bot']['owner_id']).email
else:
bot.update(event['bot'])
elif event['type'] == 'stream':
if event['op'] == 'create':
for stream in event['streams']:
if not stream['invite_only']:
stream_data = copy.deepcopy(stream)
if include_subscribers:
stream_data['subscribers'] = []
stream_data['stream_weekly_traffic'] = 0
stream_data['is_old_stream'] = False
# Add stream to never_subscribed (if not invite_only)
state['never_subscribed'].append(stream_data)
state['streams'].append(stream)
state['streams'].sort(key=lambda elt: elt["name"])
if event['op'] == 'delete':
deleted_stream_ids = {
stream['stream_id']
for stream in event['streams']
}
state['streams'] = [
s for s in state['streams']
if s['stream_id'] not in deleted_stream_ids
]
state['never_subscribed'] = [
stream for stream in state['never_subscribed']
if stream['stream_id'] not in deleted_stream_ids
]
if event['op'] == 'update':
# For legacy reasons, we call stream data 'subscriptions' in
# the state var here, for the benefit of the JS code.
for obj in state['subscriptions']:
if obj['name'].lower() == event['name'].lower():
obj[event['property']] = event['value']
# Also update the pure streams data
for stream in state['streams']:
if stream['name'].lower() == event['name'].lower():
prop = event['property']
if prop in stream:
stream[prop] = event['value']
elif event['op'] == "occupy":
state['streams'] += event['streams']
elif event['op'] == "vacate":
stream_ids = [s["stream_id"] for s in event['streams']]
state['streams'] = [
s for s in state['streams'] if s["stream_id"] not in stream_ids
]
elif event['type'] == 'default_streams':
state['realm_default_streams'] = event['default_streams']
elif event['type'] == 'default_stream_groups':
state['realm_default_stream_groups'] = event['default_stream_groups']
elif event['type'] == 'realm':
if event['op'] == "update":
field = 'realm_' + event['property']
state[field] = event['value']
# Tricky interaction: Whether we can create streams can get changed here.
if (field in [
'realm_create_stream_by_admins_only',
'realm_waiting_period_threshold'
]) and 'can_create_streams' in state:
state['can_create_streams'] = user_profile.can_create_streams()
elif event['op'] == "update_dict":
for key, value in event['data'].items():
state['realm_' + key] = value
# It's a bit messy, but this is where we need to
# update the state for whether password authentication
# is enabled on this server.
if key == 'authentication_methods':
state['realm_password_auth_enabled'] = (value['Email']
or value['LDAP'])
state['realm_email_auth_enabled'] = value['Email']
elif event['type'] == "subscription":
if not include_subscribers and event['op'] in [
'peer_add', 'peer_remove'
]:
return
if event['op'] in ["add"]:
if not include_subscribers:
# Avoid letting 'subscribers' entries end up in the list
for i, sub in enumerate(event['subscriptions']):
event['subscriptions'][i] = copy.deepcopy(
event['subscriptions'][i])
del event['subscriptions'][i]['subscribers']
def name(sub: Dict[str, Any]) -> Text:
return sub['name'].lower()
if event['op'] == "add":
added_names = set(map(name, event["subscriptions"]))
was_added = lambda s: name(s) in added_names
# add the new subscriptions
state['subscriptions'] += event['subscriptions']
# remove them from unsubscribed if they had been there
state['unsubscribed'] = [
s for s in state['unsubscribed'] if not was_added(s)
]
# remove them from never_subscribed if they had been there
state['never_subscribed'] = [
s for s in state['never_subscribed'] if not was_added(s)
]
elif event['op'] == "remove":
removed_names = set(map(name, event["subscriptions"]))
was_removed = lambda s: name(s) in removed_names
# Find the subs we are affecting.
removed_subs = list(filter(was_removed, state['subscriptions']))
# Remove our user from the subscribers of the removed subscriptions.
if include_subscribers:
for sub in removed_subs:
sub['subscribers'] = [
id for id in sub['subscribers']
if id != user_profile.id
]
# We must effectively copy the removed subscriptions from subscriptions to
# unsubscribe, since we only have the name in our data structure.
state['unsubscribed'] += removed_subs
# Now filter out the removed subscriptions from subscriptions.
state['subscriptions'] = [
s for s in state['subscriptions'] if not was_removed(s)
]
elif event['op'] == 'update':
for sub in state['subscriptions']:
if sub['name'].lower() == event['name'].lower():
sub[event['property']] = event['value']
elif event['op'] == 'peer_add':
user_id = event['user_id']
for sub in state['subscriptions']:
if (sub['name'] in event['subscriptions']
and user_id not in sub['subscribers']):
sub['subscribers'].append(user_id)
for sub in state['never_subscribed']:
if (sub['name'] in event['subscriptions']
and user_id not in sub['subscribers']):
sub['subscribers'].append(user_id)
elif event['op'] == 'peer_remove':
user_id = event['user_id']
for sub in state['subscriptions']:
if (sub['name'] in event['subscriptions']
and user_id in sub['subscribers']):
sub['subscribers'].remove(user_id)
elif event['type'] == "presence":
# TODO: Add user_id to presence update events / state format!
presence_user_profile = get_user(event['email'], user_profile.realm)
state['presences'][
event['email']] = UserPresence.get_status_dict_by_user(
presence_user_profile)[event['email']]
elif event['type'] == "update_message":
# We don't return messages in /register, so we don't need to
# do anything for content updates, but we may need to update
# the unread_msgs data if the topic of an unread message changed.
if 'subject' in event:
stream_dict = state['raw_unread_msgs']['stream_dict']
topic = event['subject']
for message_id in event['message_ids']:
if message_id in stream_dict:
stream_dict[message_id]['topic'] = topic
elif event['type'] == "delete_message":
max_message = Message.objects.filter(
usermessage__user_profile=user_profile).order_by('-id').first()
if max_message:
state['max_message_id'] = max_message.id
else:
state['max_message_id'] = -1
remove_id = event['message_id']
remove_message_id_from_unread_mgs(state, remove_id)
elif event['type'] == "reaction":
# The client will get the message with the reactions directly
pass
elif event['type'] == 'typing':
# Typing notification events are transient and thus ignored
pass
elif event['type'] == "update_message_flags":
# We don't return messages in `/register`, so most flags we
# can ignore, but we do need to update the unread_msgs data if
# unread state is changed.
if event['flag'] == 'read' and event['operation'] == 'add':
for remove_id in event['messages']:
remove_message_id_from_unread_mgs(state, remove_id)
elif event['type'] == "realm_domains":
if event['op'] == 'add':
state['realm_domains'].append(event['realm_domain'])
elif event['op'] == 'change':
for realm_domain in state['realm_domains']:
if realm_domain['domain'] == event['realm_domain']['domain']:
realm_domain['allow_subdomains'] = event['realm_domain'][
'allow_subdomains']
elif event['op'] == 'remove':
state['realm_domains'] = [
realm_domain for realm_domain in state['realm_domains']
if realm_domain['domain'] != event['domain']
]
elif event['type'] == "realm_emoji":
state['realm_emoji'] = event['realm_emoji']
elif event['type'] == "alert_words":
state['alert_words'] = event['alert_words']
elif event['type'] == "muted_topics":
state['muted_topics'] = event["muted_topics"]
elif event['type'] == "realm_filters":
state['realm_filters'] = event["realm_filters"]
elif event['type'] == "update_display_settings":
assert event['setting_name'] in UserProfile.property_types
state[event['setting_name']] = event['setting']
elif event['type'] == "update_global_notifications":
assert event[
'notification_name'] in UserProfile.notification_setting_types
state[event['notification_name']] = event['setting']
elif event['type'] == "user_group":
if event['op'] == 'add':
state['realm_user_groups'].append(event['group'])
state['realm_user_groups'].sort(key=lambda group: group['id'])
elif event['op'] == 'update':
for user_group in state['realm_user_groups']:
if user_group['id'] == event['group_id']:
user_group.update(event['data'])
elif event['op'] == 'add_members':
for user_group in state['realm_user_groups']:
if user_group['id'] == event['group_id']:
user_group['members'].extend(event['user_ids'])
user_group['members'].sort()
elif event['op'] == 'remove_members':
for user_group in state['realm_user_groups']:
if user_group['id'] == event['group_id']:
members = set(user_group['members'])
user_group['members'] = list(members -
set(event['user_ids']))
user_group['members'].sort()
elif event['op'] == 'remove':
state['realm_user_groups'] = [
ug for ug in state['realm_user_groups']
if ug['id'] != event['group_id']
]
else:
raise AssertionError("Unexpected event type %s" % (event['type'], ))
def fetch_initial_state_data(
user_profile: UserProfile,
event_types: Optional[Iterable[str]],
queue_id: str,
client_gravatar: bool,
include_subscribers: bool = True) -> Dict[str, Any]:
state = {'queue_id': queue_id} # type: Dict[str, Any]
if event_types is None:
# return True always
want = always_want # type: Callable[[str], bool]
else:
want = set(event_types).__contains__
if want('alert_words'):
state['alert_words'] = user_alert_words(user_profile)
if want('custom_profile_fields'):
fields = custom_profile_fields_for_realm(user_profile.realm.id)
state['custom_profile_fields'] = [f.as_dict() for f in fields]
state[
'custom_profile_field_types'] = CustomProfileField.FIELD_TYPE_CHOICES
if want('attachments'):
state['attachments'] = user_attachments(user_profile)
if want('hotspots'):
state['hotspots'] = get_next_hotspots(user_profile)
if want('message'):
# The client should use get_messages() to fetch messages
# starting with the max_message_id. They will get messages
# newer than that ID via get_events()
messages = Message.objects.filter(
usermessage__user_profile=user_profile).order_by('-id')[:1]
if messages:
state['max_message_id'] = messages[0].id
else:
state['max_message_id'] = -1
if want('muted_topics'):
state['muted_topics'] = get_topic_mutes(user_profile)
if want('pointer'):
state['pointer'] = user_profile.pointer
if want('presence'):
state['presences'] = get_status_dict(user_profile)
if want('realm'):
for property_name in Realm.property_types:
state['realm_' + property_name] = getattr(user_profile.realm,
property_name)
# Most state is handled via the property_types framework;
# these manual entries are for those realm settings that don't
# fit into that framework.
realm = user_profile.realm
state[
'realm_authentication_methods'] = realm.authentication_methods_dict(
)
state['realm_allow_message_editing'] = realm.allow_message_editing
state[
'realm_allow_community_topic_editing'] = realm.allow_community_topic_editing
state['realm_allow_message_deleting'] = realm.allow_message_deleting
state[
'realm_message_content_edit_limit_seconds'] = realm.message_content_edit_limit_seconds
state[
'realm_message_content_delete_limit_seconds'] = realm.message_content_delete_limit_seconds
state['realm_icon_url'] = realm_icon_url(realm)
state['realm_icon_source'] = realm.icon_source
state['max_icon_file_size'] = settings.MAX_ICON_FILE_SIZE
state['realm_bot_domain'] = realm.get_bot_domain()
state['realm_uri'] = realm.uri
state[
'realm_available_video_chat_providers'] = realm.VIDEO_CHAT_PROVIDERS
state['realm_presence_disabled'] = realm.presence_disabled
state[
'realm_show_digest_email'] = realm.show_digest_email and settings.SEND_DIGEST_EMAILS
state['realm_is_zephyr_mirror_realm'] = realm.is_zephyr_mirror_realm
state['realm_email_auth_enabled'] = email_auth_enabled(realm)
state['realm_password_auth_enabled'] = password_auth_enabled(realm)
if realm.notifications_stream and not realm.notifications_stream.deactivated:
notifications_stream = realm.notifications_stream
state['realm_notifications_stream_id'] = notifications_stream.id
else:
state['realm_notifications_stream_id'] = -1
if user_profile.realm.get_signup_notifications_stream():
signup_notifications_stream = user_profile.realm.get_signup_notifications_stream(
)
state[
'realm_signup_notifications_stream_id'] = signup_notifications_stream.id
else:
state['realm_signup_notifications_stream_id'] = -1
if want('realm_domains'):
state['realm_domains'] = get_realm_domains(user_profile.realm)
if want('realm_emoji'):
state['realm_emoji'] = user_profile.realm.get_emoji()
if want('realm_filters'):
state['realm_filters'] = realm_filters_for_realm(user_profile.realm_id)
if want('realm_user_groups'):
state['realm_user_groups'] = user_groups_in_realm_serialized(
user_profile.realm)
if want('realm_user'):
state['raw_users'] = get_raw_user_data(
realm_id=user_profile.realm_id,
client_gravatar=client_gravatar,
)
# For the user's own avatar URL, we force
# client_gravatar=False, since that saves some unnecessary
# client-side code for handing medium-size avatars. See #8253
# for details.
state['avatar_source'] = user_profile.avatar_source
state['avatar_url_medium'] = avatar_url(
user_profile,
medium=True,
client_gravatar=False,
)
state['avatar_url'] = avatar_url(
user_profile,
medium=False,
client_gravatar=False,
)
state['can_create_streams'] = user_profile.can_create_streams()
state['cross_realm_bots'] = list(get_cross_realm_dicts())
state['is_admin'] = user_profile.is_realm_admin
state['user_id'] = user_profile.id
state['enter_sends'] = user_profile.enter_sends
state['email'] = user_profile.email
state['full_name'] = user_profile.full_name
if want('realm_bot'):
state['realm_bots'] = get_owned_bot_dicts(user_profile)
# This does not yet have an apply_event counterpart, since currently,
# new entries for EMBEDDED_BOTS can only be added directly in the codebase.
if want('realm_embedded_bots'):
realm_embedded_bots = []
for bot in EMBEDDED_BOTS:
realm_embedded_bots.append({
'name':
bot.name,
'config':
load_bot_config_template(bot.name)
})
state['realm_embedded_bots'] = realm_embedded_bots
if want('subscription'):
subscriptions, unsubscribed, never_subscribed = gather_subscriptions_helper(
user_profile, include_subscribers=include_subscribers)
state['subscriptions'] = subscriptions
state['unsubscribed'] = unsubscribed
state['never_subscribed'] = never_subscribed
if want('update_message_flags') and want('message'):
# Keeping unread_msgs updated requires both message flag updates and
# message updates. This is due to the fact that new messages will not
# generate a flag update so we need to use the flags field in the
# message event.
state['raw_unread_msgs'] = get_raw_unread_data(user_profile)
if want('stream'):
state['streams'] = do_get_streams(user_profile)
state['stream_name_max_length'] = Stream.MAX_NAME_LENGTH
state['stream_description_max_length'] = Stream.MAX_DESCRIPTION_LENGTH
if want('default_streams'):
state['realm_default_streams'] = streams_to_dicts_sorted(
get_default_streams_for_realm(user_profile.realm_id))
if want('default_stream_groups'):
state[
'realm_default_stream_groups'] = default_stream_groups_to_dicts_sorted(
get_default_stream_groups(user_profile.realm))
if want('update_display_settings'):
for prop in UserProfile.property_types:
state[prop] = getattr(user_profile, prop)
state['emojiset_choices'] = user_profile.emojiset_choices()
if want('update_global_notifications'):
for notification in UserProfile.notification_setting_types:
state[notification] = getattr(user_profile, notification)
if want('zulip_version'):
state['zulip_version'] = ZULIP_VERSION
return state
def add_missing_messages(user_profile: UserProfile) -> None:
"""This function takes a soft-deactivated user, and computes and adds
to the database any UserMessage rows that were not created while
the user was soft-deactivated. The end result is that from the
perspective of the message database, it should be impossible to
tell that the user was soft-deactivated at all.
At a high level, the algorithm is as follows:
* Find all the streams that the user was at any time a subscriber
of when or after they were soft-deactivated (`recipient_ids`
below).
* Find all the messages sent to those streams since the user was
soft-deactivated. This will be a superset of the target
UserMessages we need to create in two ways: (1) some UserMessage
rows will have already been created in do_send_messages because
the user had a nonzero set of flags (the fact that we do so in
do_send_messages simplifies things considerably, since it means
we don't need to inspect message content to look for things like
mentions here), and (2) the user might not have been subscribed
to all of the streams in recipient_ids for the entire time
window.
* Correct the list from the previous state by excluding those with
existing UserMessage rows.
* Correct the list from the previous state by excluding those
where the user wasn't subscribed at the time, using the
RealmAuditLog data to determine exactly when the user was
subscribed/unsubscribed.
* Create the UserMessage rows.
For further documentation, see:
https://zulip.readthedocs.io/en/latest/subsystems/sending-messages.html#soft-deactivation
"""
assert user_profile.last_active_message_id is not None
all_stream_subs = list(
Subscription.objects.filter(user_profile=user_profile,
recipient__type=Recipient.STREAM).values(
"recipient_id", "recipient__type_id"))
# For stream messages we need to check messages against data from
# RealmAuditLog for visibility to user. So we fetch the subscription logs.
stream_ids = [sub["recipient__type_id"] for sub in all_stream_subs]
events = [
RealmAuditLog.SUBSCRIPTION_CREATED,
RealmAuditLog.SUBSCRIPTION_DEACTIVATED,
RealmAuditLog.SUBSCRIPTION_ACTIVATED,
]
# Important: We order first by event_last_message_id, which is the
# official ordering, and then tiebreak by RealmAuditLog event ID.
# That second tiebreak is important in case a user is subscribed
# and then unsubscribed without any messages being sent in the
# meantime. Without that tiebreak, we could end up incorrectly
# processing the ordering of those two subscription changes.
subscription_logs = list(
RealmAuditLog.objects.select_related("modified_stream").filter(
modified_user=user_profile,
modified_stream_id__in=stream_ids,
event_type__in=events).order_by("event_last_message_id", "id"))
all_stream_subscription_logs: DefaultDict[
int, List[RealmAuditLog]] = defaultdict(list)
for log in subscription_logs:
all_stream_subscription_logs[assert_is_not_none(
log.modified_stream_id)].append(log)
recipient_ids = []
for sub in all_stream_subs:
stream_subscription_logs = all_stream_subscription_logs[
sub["recipient__type_id"]]
if stream_subscription_logs[
-1].event_type == RealmAuditLog.SUBSCRIPTION_DEACTIVATED:
assert stream_subscription_logs[
-1].event_last_message_id is not None
if (stream_subscription_logs[-1].event_last_message_id <=
user_profile.last_active_message_id):
# We are going to short circuit this iteration as its no use
# iterating since user unsubscribed before soft-deactivation
continue
recipient_ids.append(sub["recipient_id"])
all_stream_msgs = list(
Message.objects.filter(
recipient_id__in=recipient_ids,
id__gt=user_profile.last_active_message_id).order_by("id").values(
"id", "recipient__type_id"))
already_created_ums = set(
UserMessage.objects.filter(
user_profile=user_profile,
message__recipient__type=Recipient.STREAM,
message_id__gt=user_profile.last_active_message_id,
).values_list("message_id", flat=True))
# Filter those messages for which UserMessage rows have been already created
all_stream_msgs = [
msg for msg in all_stream_msgs if msg["id"] not in already_created_ums
]
stream_messages: DefaultDict[int,
List[MissingMessageDict]] = defaultdict(list)
for msg in all_stream_msgs:
stream_messages[msg["recipient__type_id"]].append(msg)
# Calling this function to filter out stream messages based upon
# subscription logs and then store all UserMessage objects for bulk insert
# This function does not perform any SQL related task and gets all the data
# required for its operation in its params.
user_messages_to_insert = filter_by_subscription_history(
user_profile, stream_messages, all_stream_subscription_logs)
# Doing a bulk create for all the UserMessage objects stored for creation.
while len(user_messages_to_insert) > 0:
messages, user_messages_to_insert = (
user_messages_to_insert[0:BULK_CREATE_BATCH_SIZE],
user_messages_to_insert[BULK_CREATE_BATCH_SIZE:],
)
UserMessage.objects.bulk_create(messages)
user_profile.last_active_message_id = messages[-1].message_id
user_profile.save(update_fields=["last_active_message_id"])
def fetch_initial_state_data(
user_profile: Optional[UserProfile],
event_types: Optional[Iterable[str]],
queue_id: Optional[str],
client_gravatar: bool,
user_avatar_url_field_optional: bool,
realm: Realm,
slim_presence: bool = False,
include_subscribers: bool = True,
include_streams: bool = True,
) -> Dict[str, Any]:
"""When `event_types` is None, fetches the core data powering the
webapp's `page_params` and `/api/v1/register` (for mobile/terminal
apps). Can also fetch a subset as determined by `event_types`.
The user_profile=None code path is used for logged-out public
access to streams with is_web_public=True.
Whenever you add new code to this function, you should also add
corresponding events for changes in the data structures and new
code to apply_events (and add a test in test_events.py).
"""
state: Dict[str, Any] = {'queue_id': queue_id}
if event_types is None:
# return True always
want: Callable[[str], bool] = always_want
else:
want = set(event_types).__contains__
# Show the version info unconditionally.
state['zulip_version'] = ZULIP_VERSION
state['zulip_feature_level'] = API_FEATURE_LEVEL
if want('alert_words'):
state['alert_words'] = [] if user_profile is None else user_alert_words(user_profile)
if want('custom_profile_fields'):
fields = custom_profile_fields_for_realm(realm.id)
state['custom_profile_fields'] = [f.as_dict() for f in fields]
state['custom_profile_field_types'] = {
item[4]: {"id": item[0], "name": str(item[1])} for item in CustomProfileField.ALL_FIELD_TYPES
}
if want('hotspots'):
# Even if we offered special hotspots for guests without an
# account, we'd maybe need to store their state using cookies
# or local storage, rather than in the database.
state['hotspots'] = [] if user_profile is None else get_next_hotspots(user_profile)
if want('message'):
# Since the introduction of `anchor="latest"` in the API,
# `max_message_id` is primarily used for generating `local_id`
# values that are higher than this. We likely can eventually
# remove this parameter from the API.
user_messages = []
if user_profile is not None:
user_messages = UserMessage.objects \
.filter(user_profile=user_profile) \
.order_by('-message_id') \
.values('message_id')[:1]
if user_messages:
state['max_message_id'] = user_messages[0]['message_id']
else:
state['max_message_id'] = -1
if want('muted_topics'):
state['muted_topics'] = [] if user_profile is None else get_topic_mutes(user_profile)
if want('presence'):
state['presences'] = {} if user_profile is None else get_presences_for_realm(realm, slim_presence)
if want('realm'):
for property_name in Realm.property_types:
state['realm_' + property_name] = getattr(realm, property_name)
# Most state is handled via the property_types framework;
# these manual entries are for those realm settings that don't
# fit into that framework.
state['realm_authentication_methods'] = realm.authentication_methods_dict()
# We pretend these features are disabled because guests can't
# access them. In the future, we may want to move this logic
# to the frontends, so that we can correctly display what
# these fields are in the settings.
state['realm_allow_message_editing'] = False if user_profile is None else realm.allow_message_editing
state['realm_allow_community_topic_editing'] = False if user_profile is None else realm.allow_community_topic_editing
state['realm_allow_message_deleting'] = False if user_profile is None else realm.allow_message_deleting
state['realm_message_content_edit_limit_seconds'] = realm.message_content_edit_limit_seconds
state['realm_message_content_delete_limit_seconds'] = realm.message_content_delete_limit_seconds
state['realm_community_topic_editing_limit_seconds'] = \
Realm.DEFAULT_COMMUNITY_TOPIC_EDITING_LIMIT_SECONDS
# This setting determines whether to send presence and also
# whether to display of users list in the right sidebar; we
# want both behaviors for logged-out users. We may in the
# future choose to move this logic to the frontend.
state['realm_presence_disabled'] = True if user_profile is None else realm.presence_disabled
state['realm_icon_url'] = realm_icon_url(realm)
state['realm_icon_source'] = realm.icon_source
state['max_icon_file_size'] = settings.MAX_ICON_FILE_SIZE
add_realm_logo_fields(state, realm)
state['realm_bot_domain'] = realm.get_bot_domain()
state['realm_uri'] = realm.uri
state['realm_available_video_chat_providers'] = realm.VIDEO_CHAT_PROVIDERS
state['settings_send_digest_emails'] = settings.SEND_DIGEST_EMAILS
state['realm_digest_emails_enabled'] = realm.digest_emails_enabled and settings.SEND_DIGEST_EMAILS
state['realm_is_zephyr_mirror_realm'] = realm.is_zephyr_mirror_realm
state['realm_email_auth_enabled'] = email_auth_enabled(realm)
state['realm_password_auth_enabled'] = password_auth_enabled(realm)
state['realm_push_notifications_enabled'] = push_notifications_enabled()
state['realm_upload_quota'] = realm.upload_quota_bytes()
state['realm_plan_type'] = realm.plan_type
state['zulip_plan_is_not_limited'] = realm.plan_type != Realm.LIMITED
state['upgrade_text_for_wide_organization_logo'] = str(Realm.UPGRADE_TEXT_STANDARD)
state['realm_default_external_accounts'] = DEFAULT_EXTERNAL_ACCOUNTS
state['jitsi_server_url'] = settings.JITSI_SERVER_URL.rstrip('/')
state['development_environment'] = settings.DEVELOPMENT
state['server_generation'] = settings.SERVER_GENERATION
state['password_min_length'] = settings.PASSWORD_MIN_LENGTH
state['password_min_guesses'] = settings.PASSWORD_MIN_GUESSES
state['max_file_upload_size_mib'] = settings.MAX_FILE_UPLOAD_SIZE
state['max_avatar_file_size_mib'] = settings.MAX_AVATAR_FILE_SIZE
state['server_inline_image_preview'] = settings.INLINE_IMAGE_PREVIEW
state['server_inline_url_embed_preview'] = settings.INLINE_URL_EMBED_PREVIEW
state['server_avatar_changes_disabled'] = settings.AVATAR_CHANGES_DISABLED
state['server_name_changes_disabled'] = settings.NAME_CHANGES_DISABLED
if realm.notifications_stream and not realm.notifications_stream.deactivated:
notifications_stream = realm.notifications_stream
state['realm_notifications_stream_id'] = notifications_stream.id
else:
state['realm_notifications_stream_id'] = -1
signup_notifications_stream = realm.get_signup_notifications_stream()
if signup_notifications_stream:
state['realm_signup_notifications_stream_id'] = signup_notifications_stream.id
else:
state['realm_signup_notifications_stream_id'] = -1
if want('realm_domains'):
state['realm_domains'] = get_realm_domains(realm)
if want('realm_emoji'):
state['realm_emoji'] = realm.get_emoji()
if want('realm_filters'):
state['realm_filters'] = realm_filters_for_realm(realm.id)
if want('realm_user_groups'):
state['realm_user_groups'] = user_groups_in_realm_serialized(realm)
if user_profile is not None:
settings_user = user_profile
else:
# When UserProfile=None, we want to serve the values for various
# settings as the defaults. Instead of copying the default values
# from models.py here, we access these default values from a
# temporary UserProfile object that will not be saved to the database.
#
# We also can set various fields to avoid duplicating code
# unnecessarily.
settings_user = UserProfile(
full_name="Anonymous User",
email="*****@*****.**",
delivery_email="*****@*****.**",
realm=realm,
# We tag logged-out users as guests because most guest
# restrictions apply to these users as well, and it lets
# us avoid unnecessary conditionals.
role=UserProfile.ROLE_GUEST,
avatar_source=UserProfile.AVATAR_FROM_GRAVATAR,
# ID=0 is not used in real Zulip databases, ensuring this is unique.
id=0,
)
if want('realm_user'):
state['raw_users'] = get_raw_user_data(realm, user_profile,
client_gravatar=client_gravatar,
user_avatar_url_field_optional=user_avatar_url_field_optional)
state['cross_realm_bots'] = list(get_cross_realm_dicts())
# For the user's own avatar URL, we force
# client_gravatar=False, since that saves some unnecessary
# client-side code for handing medium-size avatars. See #8253
# for details.
state['avatar_source'] = settings_user.avatar_source
state['avatar_url_medium'] = avatar_url(
settings_user,
medium=True,
client_gravatar=False,
)
state['avatar_url'] = avatar_url(
settings_user,
medium=False,
client_gravatar=False,
)
state['can_create_streams'] = settings_user.can_create_streams()
state['can_subscribe_other_users'] = settings_user.can_subscribe_other_users()
state['is_admin'] = settings_user.is_realm_admin
state['is_owner'] = settings_user.is_realm_owner
state['is_guest'] = settings_user.is_guest
state['user_id'] = settings_user.id
state['enter_sends'] = settings_user.enter_sends
state['email'] = settings_user.email
state['delivery_email'] = settings_user.delivery_email
state['full_name'] = settings_user.full_name
if want('realm_bot'):
state['realm_bots'] = [] if user_profile is None else get_owned_bot_dicts(user_profile)
# This does not yet have an apply_event counterpart, since currently,
# new entries for EMBEDDED_BOTS can only be added directly in the codebase.
if want('realm_embedded_bots'):
realm_embedded_bots = []
for bot in EMBEDDED_BOTS:
realm_embedded_bots.append({'name': bot.name,
'config': load_bot_config_template(bot.name)})
state['realm_embedded_bots'] = realm_embedded_bots
# This does not have an apply_events counterpart either since
# this data is mostly static.
if want('realm_incoming_webhook_bots'):
realm_incoming_webhook_bots = []
for integration in WEBHOOK_INTEGRATIONS:
realm_incoming_webhook_bots.append({
'name': integration.name,
'config': {c[1]: c[0] for c in integration.config_options},
})
state['realm_incoming_webhook_bots'] = realm_incoming_webhook_bots
if want('recent_private_conversations'):
# A data structure containing records of this form:
#
# [{'max_message_id': 700175, 'user_ids': [801]}]
#
# for all recent private message conversations, ordered by the
# highest message ID in the conversation. The user_ids list
# is the list of users other than the current user in the
# private message conversation (so it is [] for PMs to self).
# Note that raw_recent_private_conversations is an
# intermediate form as a dictionary keyed by recipient_id,
# which is more efficient to update, and is rewritten to the
# final format in post_process_state.
state['raw_recent_private_conversations'] = {} if user_profile is None else get_recent_private_conversations(user_profile)
if want('subscription'):
if user_profile is not None:
subscriptions, unsubscribed, never_subscribed = gather_subscriptions_helper(
user_profile, include_subscribers=include_subscribers)
else:
subscriptions, unsubscribed, never_subscribed = get_web_public_subs(realm)
state['subscriptions'] = subscriptions
state['unsubscribed'] = unsubscribed
state['never_subscribed'] = never_subscribed
if want('update_message_flags') and want('message'):
# Keeping unread_msgs updated requires both message flag updates and
# message updates. This is due to the fact that new messages will not
# generate a flag update so we need to use the flags field in the
# message event.
if user_profile is not None:
state['raw_unread_msgs'] = get_raw_unread_data(user_profile)
else:
# For logged-out visitors, we treat all messages as read;
# calling this helper lets us return empty objects in the
# appropriate format.
state['raw_unread_msgs'] = extract_unread_data_from_um_rows([], user_profile)
if want('starred_messages'):
state['starred_messages'] = [] if user_profile is None else get_starred_message_ids(user_profile)
if want('stream'):
if include_streams:
# The webapp doesn't use the data from here; instead,
# it uses data from state["subscriptions"] and other
# places.
if user_profile is not None:
state['streams'] = do_get_streams(user_profile)
else:
state['streams'] = get_web_public_streams(realm)
state['stream_name_max_length'] = Stream.MAX_NAME_LENGTH
state['stream_description_max_length'] = Stream.MAX_DESCRIPTION_LENGTH
if want('default_streams'):
if settings_user.is_guest:
# Guest users and logged-out users don't have access to
# all default streams, so we pretend the organization
# doesn't have any.
state['realm_default_streams'] = []
else:
state['realm_default_streams'] = streams_to_dicts_sorted(
get_default_streams_for_realm(realm.id))
if want('default_stream_groups'):
if settings_user.is_guest:
state['realm_default_stream_groups'] = []
else:
state['realm_default_stream_groups'] = default_stream_groups_to_dicts_sorted(
get_default_stream_groups(realm))
if want('stop_words'):
state['stop_words'] = read_stop_words()
if want('update_display_settings'):
for prop in UserProfile.property_types:
state[prop] = getattr(settings_user, prop)
state['emojiset_choices'] = UserProfile.emojiset_choices()
if want('update_global_notifications'):
for notification in UserProfile.notification_setting_types:
state[notification] = getattr(settings_user, notification)
state['available_notification_sounds'] = get_available_notification_sounds()
if want('user_status'):
# We require creating an account to access statuses.
state['user_status'] = {} if user_profile is None else get_user_info_dict(realm_id=realm.id)
if want('video_calls'):
state['has_zoom_token'] = settings_user.zoom_token is not None
return state
ctx = {
"new_email_html_tag":
SafeString(
f'<a href="mailto:{escape(new_email)}">{escape(new_email)}</a>'),
"old_email_html_tag":
SafeString(
f'<a href="mailto:{escape(old_email)}">{escape(old_email)}</a>'),
}
return render(request,
"confirmation/confirm_email_change.html",
context=ctx)
emojiset_choices = {
emojiset["key"]
for emojiset in UserProfile.emojiset_choices()
}
default_view_options = ["recent_topics", "all_messages"]
def check_settings_values(
notification_sound: Optional[str],
email_notifications_batching_period_seconds: Optional[int],
default_language: Optional[str] = None,
) -> None:
# We can't use REQ for this widget because
# get_available_language_codes requires provisioning to be
# complete.
if default_language is not None and default_language not in get_available_language_codes(
):
raise JsonableError(_("Invalid default_language"))
def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile,
missed_messages: List[Message],
message_count: int) -> None:
"""
Send a reminder email to a user if she's missed some PMs by being offline.
The email will have its reply to address set to a limited used email
address that will send a zulip message to the correct recipient. This
allows the user to respond to missed PMs, huddles, and @-mentions directly
from the email.
`user_profile` is the user to send the reminder to
`missed_messages` is a list of Message objects to remind about they should
all have the same recipient and subject
"""
from zerver.context_processors import common_context
# Disabled missedmessage emails internally
if not user_profile.enable_offline_email_notifications:
return
recipients = set((msg.recipient_id, msg.subject) for msg in missed_messages)
if len(recipients) != 1:
raise ValueError(
'All missed_messages must have the same recipient and subject %r' %
recipients
)
unsubscribe_link = one_click_unsubscribe_link(user_profile, "missed_messages")
context = common_context(user_profile)
context.update({
'name': user_profile.full_name,
'message_count': message_count,
'mention': missed_messages[0].is_stream_message(),
'unsubscribe_link': unsubscribe_link,
'realm_name_in_notifications': user_profile.realm_name_in_notifications,
'show_message_content': user_profile.message_content_in_email_notifications,
})
# If this setting (email mirroring integration) is enabled, only then
# can users reply to email to send message to Zulip. Thus, one must
# ensure to display warning in the template.
if settings.EMAIL_GATEWAY_PATTERN:
context.update({
'reply_warning': False,
'reply_to_zulip': True,
})
else:
context.update({
'reply_warning': True,
'reply_to_zulip': False,
})
from zerver.lib.email_mirror import create_missed_message_address
reply_to_address = create_missed_message_address(user_profile, missed_messages[0])
if reply_to_address == FromAddress.NOREPLY:
reply_to_name = None
else:
reply_to_name = "Zulip"
senders = list(set(m.sender for m in missed_messages))
if (missed_messages[0].recipient.type == Recipient.HUDDLE):
display_recipient = get_display_recipient(missed_messages[0].recipient)
# Make sure that this is a list of strings, not a string.
assert not isinstance(display_recipient, str)
other_recipients = [r['full_name'] for r in display_recipient
if r['id'] != user_profile.id]
context.update({'group_pm': True})
if len(other_recipients) == 2:
huddle_display_name = "%s" % (" and ".join(other_recipients))
context.update({'huddle_display_name': huddle_display_name})
elif len(other_recipients) == 3:
huddle_display_name = "%s, %s, and %s" % (
other_recipients[0], other_recipients[1], other_recipients[2])
context.update({'huddle_display_name': huddle_display_name})
else:
huddle_display_name = "%s, and %s others" % (
', '.join(other_recipients[:2]), len(other_recipients) - 2)
context.update({'huddle_display_name': huddle_display_name})
elif (missed_messages[0].recipient.type == Recipient.PERSONAL):
context.update({'private_message': True})
else:
# Keep only the senders who actually mentioned the user
#
# TODO: When we add wildcard mentions that send emails, add
# them to the filter here.
senders = list(set(m.sender for m in missed_messages if
UserMessage.objects.filter(message=m, user_profile=user_profile,
flags=UserMessage.flags.mentioned).exists()))
context.update({'at_mention': True})
# If message content is disabled, then flush all information we pass to email.
if not user_profile.message_content_in_email_notifications:
context.update({
'reply_to_zulip': False,
'messages': [],
'sender_str': "",
'realm_str': user_profile.realm.name,
'huddle_display_name': "",
})
else:
context.update({
'messages': build_message_list(user_profile, missed_messages),
'sender_str': ", ".join(sender.full_name for sender in senders),
'realm_str': user_profile.realm.name,
})
from_name = "Zulip missed messages" # type: str
from_address = FromAddress.NOREPLY
if len(senders) == 1 and settings.SEND_MISSED_MESSAGE_EMAILS_AS_USER:
# If this setting is enabled, you can reply to the Zulip
# missed message emails directly back to the original sender.
# However, one must ensure the Zulip server is in the SPF
# record for the domain, or there will be spam/deliverability
# problems.
sender = senders[0]
from_name, from_address = (sender.full_name, sender.email)
context.update({
'reply_warning': False,
'reply_to_zulip': False,
})
email_dict = {
'template_prefix': 'zerver/emails/missed_message',
'to_user_id': user_profile.id,
'from_name': from_name,
'from_address': from_address,
'reply_to_email': formataddr((reply_to_name, reply_to_address)),
'context': context}
queue_json_publish("email_senders", email_dict)
user_profile.last_reminder = timezone_now()
user_profile.save(update_fields=['last_reminder'])
def get_display_email_address(user_profile: UserProfile, realm: Realm) -> str:
if not user_profile.email_address_is_realm_public():
return f"user{user_profile.id}@{get_fake_email_domain()}"
return user_profile.delivery_email
def fetch_initial_state_data(
user_profile: UserProfile,
event_types: Optional[Iterable[str]],
queue_id: str,
client_gravatar: bool,
slim_presence: bool = False,
include_subscribers: bool = True) -> Dict[str, Any]:
state = {'queue_id': queue_id} # type: Dict[str, Any]
realm = user_profile.realm
if event_types is None:
# return True always
want = always_want # type: Callable[[str], bool]
else:
want = set(event_types).__contains__
if want('alert_words'):
state['alert_words'] = user_alert_words(user_profile)
if want('custom_profile_fields'):
fields = custom_profile_fields_for_realm(realm.id)
state['custom_profile_fields'] = [f.as_dict() for f in fields]
state[
'custom_profile_field_types'] = CustomProfileField.FIELD_TYPE_CHOICES_DICT
if want('hotspots'):
state['hotspots'] = get_next_hotspots(user_profile)
if want('message'):
# The client should use get_messages() to fetch messages
# starting with the max_message_id. They will get messages
# newer than that ID via get_events()
messages = Message.objects.filter(
usermessage__user_profile=user_profile).order_by('-id')[:1]
if messages:
state['max_message_id'] = messages[0].id
else:
state['max_message_id'] = -1
if want('muted_topics'):
state['muted_topics'] = get_topic_mutes(user_profile)
if want('pointer'):
state['pointer'] = user_profile.pointer
if want('presence'):
state['presences'] = get_presences_for_realm(realm, slim_presence)
if want('realm'):
for property_name in Realm.property_types:
state['realm_' + property_name] = getattr(realm, property_name)
# Don't send the zoom API secret to clients.
if state.get('realm_zoom_api_secret'):
state['realm_zoom_api_secret'] = ''
# Most state is handled via the property_types framework;
# these manual entries are for those realm settings that don't
# fit into that framework.
state[
'realm_authentication_methods'] = realm.authentication_methods_dict(
)
state['realm_allow_message_editing'] = realm.allow_message_editing
state[
'realm_allow_community_topic_editing'] = realm.allow_community_topic_editing
state['realm_allow_message_deleting'] = realm.allow_message_deleting
state[
'realm_message_content_edit_limit_seconds'] = realm.message_content_edit_limit_seconds
state[
'realm_message_content_delete_limit_seconds'] = realm.message_content_delete_limit_seconds
state['realm_icon_url'] = realm_icon_url(realm)
state['realm_icon_source'] = realm.icon_source
state['max_icon_file_size'] = settings.MAX_ICON_FILE_SIZE
add_realm_logo_fields(state, realm)
state['realm_bot_domain'] = realm.get_bot_domain()
state['realm_uri'] = realm.uri
state[
'realm_available_video_chat_providers'] = realm.VIDEO_CHAT_PROVIDERS
state['realm_presence_disabled'] = realm.presence_disabled
state['settings_send_digest_emails'] = settings.SEND_DIGEST_EMAILS
state[
'realm_digest_emails_enabled'] = realm.digest_emails_enabled and settings.SEND_DIGEST_EMAILS
state['realm_is_zephyr_mirror_realm'] = realm.is_zephyr_mirror_realm
state['realm_email_auth_enabled'] = email_auth_enabled(realm)
state['realm_password_auth_enabled'] = password_auth_enabled(realm)
state['realm_push_notifications_enabled'] = push_notifications_enabled(
)
state['realm_upload_quota'] = realm.upload_quota_bytes()
state['realm_plan_type'] = realm.plan_type
state[
'plan_includes_wide_organization_logo'] = realm.plan_type != Realm.LIMITED
state['upgrade_text_for_wide_organization_logo'] = str(
Realm.UPGRADE_TEXT_STANDARD)
state['realm_default_external_accounts'] = DEFAULT_EXTERNAL_ACCOUNTS
if realm.notifications_stream and not realm.notifications_stream.deactivated:
notifications_stream = realm.notifications_stream
state['realm_notifications_stream_id'] = notifications_stream.id
else:
state['realm_notifications_stream_id'] = -1
signup_notifications_stream = realm.get_signup_notifications_stream()
if signup_notifications_stream:
state[
'realm_signup_notifications_stream_id'] = signup_notifications_stream.id
else:
state['realm_signup_notifications_stream_id'] = -1
if want('realm_domains'):
state['realm_domains'] = get_realm_domains(realm)
if want('realm_emoji'):
state['realm_emoji'] = realm.get_emoji()
if want('realm_filters'):
state['realm_filters'] = realm_filters_for_realm(realm.id)
if want('realm_user_groups'):
state['realm_user_groups'] = user_groups_in_realm_serialized(realm)
if want('realm_user'):
state['raw_users'] = get_raw_user_data(realm,
user_profile,
client_gravatar=client_gravatar)
# For the user's own avatar URL, we force
# client_gravatar=False, since that saves some unnecessary
# client-side code for handing medium-size avatars. See #8253
# for details.
state['avatar_source'] = user_profile.avatar_source
state['avatar_url_medium'] = avatar_url(
user_profile,
medium=True,
client_gravatar=False,
)
state['avatar_url'] = avatar_url(
user_profile,
medium=False,
client_gravatar=False,
)
state['can_create_streams'] = user_profile.can_create_streams()
state[
'can_subscribe_other_users'] = user_profile.can_subscribe_other_users(
)
state['cross_realm_bots'] = list(get_cross_realm_dicts())
state['is_admin'] = user_profile.is_realm_admin
state['is_guest'] = user_profile.is_guest
state['user_id'] = user_profile.id
state['enter_sends'] = user_profile.enter_sends
state['email'] = user_profile.email
state['delivery_email'] = user_profile.delivery_email
state['full_name'] = user_profile.full_name
if want('realm_bot'):
state['realm_bots'] = get_owned_bot_dicts(user_profile)
# This does not yet have an apply_event counterpart, since currently,
# new entries for EMBEDDED_BOTS can only be added directly in the codebase.
if want('realm_embedded_bots'):
realm_embedded_bots = []
for bot in EMBEDDED_BOTS:
realm_embedded_bots.append({
'name':
bot.name,
'config':
load_bot_config_template(bot.name)
})
state['realm_embedded_bots'] = realm_embedded_bots
# This does not have an apply_events counterpart either since
# this data is mostly static.
if want('realm_incoming_webhook_bots'):
realm_incoming_webhook_bots = []
for integration in WEBHOOK_INTEGRATIONS:
realm_incoming_webhook_bots.append({
'name': integration.name,
'config': {c[1]: c[0]
for c in integration.config_options}
})
state['realm_incoming_webhook_bots'] = realm_incoming_webhook_bots
if want('recent_private_conversations'):
# A data structure containing records of this form:
#
# [{'max_message_id': 700175, 'user_ids': [801]}]
#
# for all recent private message conversations, ordered by the
# highest message ID in the conversation. The user_ids list
# is the list of users other than the current user in the
# private message conversation (so it is [] for PMs to self).
# Note that raw_recent_private_conversations is an
# intermediate form as a dictionary keyed by recipient_id,
# which is more efficient to update, and is rewritten to the
# final format in post_process_state.
state[
'raw_recent_private_conversations'] = get_recent_private_conversations(
user_profile)
if want('subscription'):
subscriptions, unsubscribed, never_subscribed = gather_subscriptions_helper(
user_profile, include_subscribers=include_subscribers)
state['subscriptions'] = subscriptions
state['unsubscribed'] = unsubscribed
state['never_subscribed'] = never_subscribed
if want('update_message_flags') and want('message'):
# Keeping unread_msgs updated requires both message flag updates and
# message updates. This is due to the fact that new messages will not
# generate a flag update so we need to use the flags field in the
# message event.
state['raw_unread_msgs'] = get_raw_unread_data(user_profile)
if want('starred_messages'):
state['starred_messages'] = get_starred_message_ids(user_profile)
if want('stream'):
state['streams'] = do_get_streams(user_profile)
state['stream_name_max_length'] = Stream.MAX_NAME_LENGTH
state['stream_description_max_length'] = Stream.MAX_DESCRIPTION_LENGTH
if want('default_streams'):
if user_profile.is_guest:
state['realm_default_streams'] = []
else:
state['realm_default_streams'] = streams_to_dicts_sorted(
get_default_streams_for_realm(realm.id))
if want('default_stream_groups'):
if user_profile.is_guest:
state['realm_default_stream_groups'] = []
else:
state[
'realm_default_stream_groups'] = default_stream_groups_to_dicts_sorted(
get_default_stream_groups(realm))
if want('stop_words'):
state['stop_words'] = read_stop_words()
if want('update_display_settings'):
for prop in UserProfile.property_types:
state[prop] = getattr(user_profile, prop)
state['emojiset_choices'] = user_profile.emojiset_choices()
if want('update_global_notifications'):
for notification in UserProfile.notification_setting_types:
state[notification] = getattr(user_profile, notification)
state[
'available_notification_sounds'] = get_available_notification_sounds(
)
if want('user_status'):
state['user_status'] = get_user_info_dict(realm_id=realm.id)
if want('zulip_version'):
state['zulip_version'] = ZULIP_VERSION
return state
def apply_event(state: Dict[str, Any],
event: Dict[str, Any],
user_profile: UserProfile,
client_gravatar: bool,
include_subscribers: bool) -> None:
if event['type'] == "message":
state['max_message_id'] = max(state['max_message_id'], event['message']['id'])
if 'raw_unread_msgs' in state:
apply_unread_message_event(
user_profile,
state['raw_unread_msgs'],
event['message'],
event['flags'],
)
elif event['type'] == "hotspots":
state['hotspots'] = event['hotspots']
elif event['type'] == "custom_profile_fields":
state['custom_profile_fields'] = event['fields']
elif event['type'] == "pointer":
state['pointer'] = max(state['pointer'], event['pointer'])
elif event['type'] == "realm_user":
person = event['person']
person_user_id = person['user_id']
if event['op'] == "add":
person = copy.deepcopy(person)
if client_gravatar:
if 'gravatar.com' in person['avatar_url']:
person['avatar_url'] = None
person['is_active'] = True
if not person['is_bot']:
person['profile_data'] = {}
state['raw_users'][person_user_id] = person
elif event['op'] == "remove":
state['raw_users'][person_user_id]['is_active'] = False
elif event['op'] == 'update':
is_me = (person_user_id == user_profile.id)
if is_me:
if ('avatar_url' in person and 'avatar_url' in state):
state['avatar_source'] = person['avatar_source']
state['avatar_url'] = person['avatar_url']
state['avatar_url_medium'] = person['avatar_url_medium']
for field in ['is_admin', 'delivery_email', 'email', 'full_name']:
if field in person and field in state:
state[field] = person[field]
# In the unlikely event that the current user
# just changed to/from being an admin, we need
# to add/remove the data on all bots in the
# realm. This is ugly and probably better
# solved by removing the all-realm-bots data
# given to admin users from this flow.
if ('is_admin' in person and 'realm_bots' in state):
prev_state = state['raw_users'][user_profile.id]
was_admin = prev_state['is_admin']
now_admin = person['is_admin']
if was_admin and not now_admin:
state['realm_bots'] = []
if not was_admin and now_admin:
state['realm_bots'] = get_owned_bot_dicts(user_profile)
if client_gravatar and 'avatar_url' in person:
# Respect the client_gravatar setting in the `users` data.
if 'gravatar.com' in person['avatar_url']:
person['avatar_url'] = None
person['avatar_url_medium'] = None
if person_user_id in state['raw_users']:
p = state['raw_users'][person_user_id]
for field in p:
if field in person:
p[field] = person[field]
if 'custom_profile_field' in person:
custom_field_id = person['custom_profile_field']['id']
custom_field_new_value = person['custom_profile_field']['value']
if 'rendered_value' in person['custom_profile_field']:
p['profile_data'][custom_field_id] = {
'value': custom_field_new_value,
'rendered_value': person['custom_profile_field']['rendered_value']
}
else:
p['profile_data'][custom_field_id] = {
'value': custom_field_new_value
}
elif event['type'] == 'realm_bot':
if event['op'] == 'add':
state['realm_bots'].append(event['bot'])
if event['op'] == 'remove':
email = event['bot']['email']
for bot in state['realm_bots']:
if bot['email'] == email:
bot['is_active'] = False
if event['op'] == 'delete':
state['realm_bots'] = [item for item
in state['realm_bots'] if item['email'] != event['bot']['email']]
if event['op'] == 'update':
for bot in state['realm_bots']:
if bot['email'] == event['bot']['email']:
if 'owner_id' in event['bot']:
bot['owner'] = get_user_profile_by_id(event['bot']['owner_id']).email
else:
bot.update(event['bot'])
elif event['type'] == 'stream':
if event['op'] == 'create':
for stream in event['streams']:
if not stream['invite_only']:
stream_data = copy.deepcopy(stream)
if include_subscribers:
stream_data['subscribers'] = []
stream_data['stream_weekly_traffic'] = None
stream_data['is_old_stream'] = False
stream_data['is_announcement_only'] = False
# Add stream to never_subscribed (if not invite_only)
state['never_subscribed'].append(stream_data)
state['streams'].append(stream)
state['streams'].sort(key=lambda elt: elt["name"])
if event['op'] == 'delete':
deleted_stream_ids = {stream['stream_id'] for stream in event['streams']}
state['streams'] = [s for s in state['streams'] if s['stream_id'] not in deleted_stream_ids]
state['never_subscribed'] = [stream for stream in state['never_subscribed'] if
stream['stream_id'] not in deleted_stream_ids]
if event['op'] == 'update':
# For legacy reasons, we call stream data 'subscriptions' in
# the state var here, for the benefit of the JS code.
for obj in state['subscriptions']:
if obj['name'].lower() == event['name'].lower():
obj[event['property']] = event['value']
# Also update the pure streams data
for stream in state['streams']:
if stream['name'].lower() == event['name'].lower():
prop = event['property']
if prop in stream:
stream[prop] = event['value']
elif event['op'] == "occupy":
state['streams'] += event['streams']
elif event['op'] == "vacate":
stream_ids = [s["stream_id"] for s in event['streams']]
state['streams'] = [s for s in state['streams'] if s["stream_id"] not in stream_ids]
elif event['type'] == 'default_streams':
state['realm_default_streams'] = event['default_streams']
elif event['type'] == 'default_stream_groups':
state['realm_default_stream_groups'] = event['default_stream_groups']
elif event['type'] == 'realm':
if event['op'] == "update":
field = 'realm_' + event['property']
state[field] = event['value']
# Tricky interaction: Whether we can create streams can get changed here.
if (field in ['realm_create_stream_by_admins_only',
'realm_waiting_period_threshold']) and 'can_create_streams' in state:
state['can_create_streams'] = user_profile.can_create_streams()
state['can_subscribe_other_users'] = user_profile.can_subscribe_other_users()
elif event['op'] == "update_dict":
for key, value in event['data'].items():
state['realm_' + key] = value
# It's a bit messy, but this is where we need to
# update the state for whether password authentication
# is enabled on this server.
if key == 'authentication_methods':
state['realm_password_auth_enabled'] = (value['Email'] or value['LDAP'])
state['realm_email_auth_enabled'] = value['Email']
elif event['type'] == "subscription":
if not include_subscribers and event['op'] in ['peer_add', 'peer_remove']:
return
if event['op'] in ["add"]:
if not include_subscribers:
# Avoid letting 'subscribers' entries end up in the list
for i, sub in enumerate(event['subscriptions']):
event['subscriptions'][i] = copy.deepcopy(event['subscriptions'][i])
del event['subscriptions'][i]['subscribers']
def name(sub: Dict[str, Any]) -> str:
return sub['name'].lower()
if event['op'] == "add":
added_names = set(map(name, event["subscriptions"]))
was_added = lambda s: name(s) in added_names
# add the new subscriptions
state['subscriptions'] += event['subscriptions']
# remove them from unsubscribed if they had been there
state['unsubscribed'] = [s for s in state['unsubscribed'] if not was_added(s)]
# remove them from never_subscribed if they had been there
state['never_subscribed'] = [s for s in state['never_subscribed'] if not was_added(s)]
elif event['op'] == "remove":
removed_names = set(map(name, event["subscriptions"]))
was_removed = lambda s: name(s) in removed_names
# Find the subs we are affecting.
removed_subs = list(filter(was_removed, state['subscriptions']))
# Remove our user from the subscribers of the removed subscriptions.
if include_subscribers:
for sub in removed_subs:
sub['subscribers'] = [id for id in sub['subscribers'] if id != user_profile.id]
# We must effectively copy the removed subscriptions from subscriptions to
# unsubscribe, since we only have the name in our data structure.
state['unsubscribed'] += removed_subs
# Now filter out the removed subscriptions from subscriptions.
state['subscriptions'] = [s for s in state['subscriptions'] if not was_removed(s)]
elif event['op'] == 'update':
for sub in state['subscriptions']:
if sub['name'].lower() == event['name'].lower():
sub[event['property']] = event['value']
elif event['op'] == 'peer_add':
user_id = event['user_id']
for sub in state['subscriptions']:
if (sub['name'] in event['subscriptions'] and
user_id not in sub['subscribers']):
sub['subscribers'].append(user_id)
for sub in state['never_subscribed']:
if (sub['name'] in event['subscriptions'] and
user_id not in sub['subscribers']):
sub['subscribers'].append(user_id)
elif event['op'] == 'peer_remove':
user_id = event['user_id']
for sub in state['subscriptions']:
if (sub['name'] in event['subscriptions'] and
user_id in sub['subscribers']):
sub['subscribers'].remove(user_id)
elif event['type'] == "presence":
# TODO: Add user_id to presence update events / state format!
presence_user_profile = get_user(event['email'], user_profile.realm)
state['presences'][event['email']] = UserPresence.get_status_dict_by_user(
presence_user_profile)[event['email']]
elif event['type'] == "update_message":
# We don't return messages in /register, so we don't need to
# do anything for content updates, but we may need to update
# the unread_msgs data if the topic of an unread message changed.
if TOPIC_NAME in event:
stream_dict = state['raw_unread_msgs']['stream_dict']
topic = event[TOPIC_NAME]
for message_id in event['message_ids']:
if message_id in stream_dict:
stream_dict[message_id]['topic'] = topic
elif event['type'] == "delete_message":
max_message = Message.objects.filter(
usermessage__user_profile=user_profile).order_by('-id').first()
if max_message:
state['max_message_id'] = max_message.id
else:
state['max_message_id'] = -1
remove_id = event['message_id']
remove_message_id_from_unread_mgs(state, remove_id)
elif event['type'] == "reaction":
# The client will get the message with the reactions directly
pass
elif event['type'] == "submessage":
# The client will get submessages with their messages
pass
elif event['type'] == 'typing':
# Typing notification events are transient and thus ignored
pass
elif event['type'] == "attachment":
# Attachment events are just for updating the "uploads" UI;
# they are not sent directly.
pass
elif event['type'] == "update_message_flags":
# We don't return messages in `/register`, so most flags we
# can ignore, but we do need to update the unread_msgs data if
# unread state is changed.
if event['flag'] == 'read' and event['operation'] == 'add':
for remove_id in event['messages']:
remove_message_id_from_unread_mgs(state, remove_id)
if event['flag'] == 'starred' and event['operation'] == 'add':
state['starred_messages'] += event['messages']
if event['flag'] == 'starred' and event['operation'] == 'remove':
state['starred_messages'] = [message for message in state['starred_messages']
if not (message in event['messages'])]
elif event['type'] == "realm_domains":
if event['op'] == 'add':
state['realm_domains'].append(event['realm_domain'])
elif event['op'] == 'change':
for realm_domain in state['realm_domains']:
if realm_domain['domain'] == event['realm_domain']['domain']:
realm_domain['allow_subdomains'] = event['realm_domain']['allow_subdomains']
elif event['op'] == 'remove':
state['realm_domains'] = [realm_domain for realm_domain in state['realm_domains']
if realm_domain['domain'] != event['domain']]
elif event['type'] == "realm_emoji":
state['realm_emoji'] = event['realm_emoji']
elif event['type'] == "alert_words":
state['alert_words'] = event['alert_words']
elif event['type'] == "muted_topics":
state['muted_topics'] = event["muted_topics"]
elif event['type'] == "realm_filters":
state['realm_filters'] = event["realm_filters"]
elif event['type'] == "update_display_settings":
assert event['setting_name'] in UserProfile.property_types
state[event['setting_name']] = event['setting']
elif event['type'] == "update_global_notifications":
assert event['notification_name'] in UserProfile.notification_setting_types
state[event['notification_name']] = event['setting']
elif event['type'] == "invites_changed":
pass
elif event['type'] == "user_group":
if event['op'] == 'add':
state['realm_user_groups'].append(event['group'])
state['realm_user_groups'].sort(key=lambda group: group['id'])
elif event['op'] == 'update':
for user_group in state['realm_user_groups']:
if user_group['id'] == event['group_id']:
user_group.update(event['data'])
elif event['op'] == 'add_members':
for user_group in state['realm_user_groups']:
if user_group['id'] == event['group_id']:
user_group['members'].extend(event['user_ids'])
user_group['members'].sort()
elif event['op'] == 'remove_members':
for user_group in state['realm_user_groups']:
if user_group['id'] == event['group_id']:
members = set(user_group['members'])
user_group['members'] = list(members - set(event['user_ids']))
user_group['members'].sort()
elif event['op'] == 'remove':
state['realm_user_groups'] = [ug for ug in state['realm_user_groups']
if ug['id'] != event['group_id']]
elif event['type'] == 'user_status':
away_user_ids = set(state['away_user_ids'])
user_id = event['user_id']
if event['away']:
away_user_ids.add(user_id)
else:
away_user_ids.discard(user_id)
state['away_user_ids'] = sorted(list(away_user_ids))
else:
raise AssertionError("Unexpected event type %s" % (event['type'],))
do_start_email_change_process(user_profile, new_email)
result['account_email'] = _("Check your email for a confirmation link. ")
if user_profile.full_name != full_name and full_name.strip() != "":
if name_changes_disabled(user_profile.realm) and not user_profile.is_realm_admin:
# Failingly silently is fine -- they can't do it through the UI, so
# they'd have to be trying to break the rules.
pass
else:
# Note that check_change_full_name strips the passed name automatically
result['full_name'] = check_change_full_name(user_profile, full_name, user_profile)
return json_success(result)
all_timezones = set(get_all_timezones())
emojiset_choices = {emojiset['key'] for emojiset in UserProfile.emojiset_choices()}
@human_users_only
@has_request_variables
def update_display_settings_backend(
request: HttpRequest, user_profile: UserProfile,
twenty_four_hour_time: Optional[bool]=REQ(validator=check_bool, default=None),
dense_mode: Optional[bool]=REQ(validator=check_bool, default=None),
starred_message_counts: Optional[bool]=REQ(validator=check_bool, default=None),
fluid_layout_width: Optional[bool]=REQ(validator=check_bool, default=None),
high_contrast_mode: Optional[bool]=REQ(validator=check_bool, default=None),
color_scheme: Optional[int]=REQ(validator=check_int_in(
UserProfile.COLOR_SCHEME_CHOICES), default=None),
translate_emoticons: Optional[bool]=REQ(validator=check_bool, default=None),
default_language: Optional[str]=REQ(validator=check_string, default=None),
left_side_userlist: Optional[bool]=REQ(validator=check_bool, default=None),
def do_change_bot_owner(user_profile: UserProfile, bot_owner: UserProfile,
acting_user: UserProfile) -> None:
previous_owner = user_profile.bot_owner
user_profile.bot_owner = bot_owner
user_profile.save(
) # Can't use update_fields because of how the foreign key works.
event_time = timezone_now()
RealmAuditLog.objects.create(
realm=user_profile.realm,
acting_user=acting_user,
modified_user=user_profile,
event_type=RealmAuditLog.USER_BOT_OWNER_CHANGED,
event_time=event_time,
)
update_users = bot_owner_user_ids(user_profile)
# For admins, update event is sent instead of delete/add
# event. bot_data of admin contains all the
# bots and none of them should be removed/(added again).
# Delete the bot from previous owner's bot data.
if previous_owner and not previous_owner.is_realm_admin:
delete_event = dict(
type="realm_bot",
op="delete",
bot=dict(user_id=user_profile.id, ),
)
transaction.on_commit(lambda: send_event(
user_profile.realm,
delete_event,
{previous_owner.id},
))
# Do not send update event for previous bot owner.
update_users = update_users - {previous_owner.id}
# Notify the new owner that the bot has been added.
if not bot_owner.is_realm_admin:
add_event = created_bot_event(user_profile)
transaction.on_commit(
lambda: send_event(user_profile.realm, add_event, {bot_owner.id}))
# Do not send update event for bot_owner.
update_users = update_users - {bot_owner.id}
bot_event = dict(
type="realm_bot",
op="update",
bot=dict(
user_id=user_profile.id,
owner_id=user_profile.bot_owner.id,
),
)
transaction.on_commit(lambda: send_event(
user_profile.realm,
bot_event,
update_users,
))
# Since `bot_owner_id` is included in the user profile dict we need
# to update the users dict with the new bot owner id
event = dict(
type="realm_user",
op="update",
person=dict(
user_id=user_profile.id,
bot_owner_id=user_profile.bot_owner.id,
),
)
transaction.on_commit(lambda: send_event(
user_profile.realm, event, active_user_ids(user_profile.realm_id)))
def patch_bot_backend(
request: HttpRequest, user_profile: UserProfile, email: Text,
full_name: Optional[Text]=REQ(default=None),
bot_owner: Optional[Text]=REQ(default=None),
default_sending_stream: Optional[Text]=REQ(default=None),
default_events_register_stream: Optional[Text]=REQ(default=None),
default_all_public_streams: Optional[bool]=REQ(default=None, validator=check_bool)
) -> HttpResponse:
try:
bot = get_user(email, user_profile.realm)
except UserProfile.DoesNotExist:
return json_error(_('No such user'))
if not user_profile.can_admin_user(bot):
return json_error(_('Insufficient permission'))
if full_name is not None:
check_change_full_name(bot, full_name, user_profile)
if bot_owner is not None:
owner = get_user(bot_owner, user_profile.realm)
do_change_bot_owner(bot, owner, user_profile)
if default_sending_stream is not None:
if default_sending_stream == "":
stream = None # type: Optional[Stream]
else:
(stream, recipient, sub) = access_stream_by_name(
user_profile, default_sending_stream)
do_change_default_sending_stream(bot, stream)
if default_events_register_stream is not None:
if default_events_register_stream == "":
stream = None
else:
(stream, recipient, sub) = access_stream_by_name(
user_profile, default_events_register_stream)
do_change_default_events_register_stream(bot, stream)
if default_all_public_streams is not None:
do_change_default_all_public_streams(bot, default_all_public_streams)
if len(request.FILES) == 0:
pass
elif len(request.FILES) == 1:
user_file = list(request.FILES.values())[0]
upload_avatar_image(user_file, user_profile, bot)
avatar_source = UserProfile.AVATAR_FROM_USER
do_change_avatar_fields(bot, avatar_source)
else:
return json_error(_("You may only upload one file at a time"))
json_result = dict(
full_name=bot.full_name,
avatar_url=avatar_url(bot),
default_sending_stream=get_stream_name(bot.default_sending_stream),
default_events_register_stream=get_stream_name(bot.default_events_register_stream),
default_all_public_streams=bot.default_all_public_streams,
)
# Don't include the bot owner in case it is not set.
# Default bots have no owner.
if bot.bot_owner is not None:
json_result['bot_owner'] = bot.bot_owner.email
return json_success(json_result)
def set_user_alert_words(user_profile: UserProfile, alert_words: List[str]) -> None:
user_profile.alert_words = ujson.dumps(alert_words)
user_profile.save(update_fields=['alert_words'])
def _wrapped_view_func(request: HttpRequest, user_profile: UserProfile,
*args: object, **kwargs: object) -> HttpResponse:
if not user_profile.can_edit_user_groups():
raise JsonableError(_("Insufficient permission"))
return view_func(request, user_profile, *args, **kwargs)
def do_send_missedmessage_events_reply_in_zulip(user_profile: UserProfile,
missed_messages: List[Dict[
str, Any]],
message_count: int) -> None:
"""
Send a reminder email to a user if she's missed some PMs by being offline.
The email will have its reply to address set to a limited used email
address that will send a zulip message to the correct recipient. This
allows the user to respond to missed PMs, huddles, and @-mentions directly
from the email.
`user_profile` is the user to send the reminder to
`missed_messages` is a list of dictionaries to Message objects and other data
for a group of messages that share a recipient (and topic)
"""
from zerver.context_processors import common_context
# Disabled missedmessage emails internally
if not user_profile.enable_offline_email_notifications:
return
recipients = {(msg['message'].recipient_id, msg['message'].topic_name())
for msg in missed_messages}
if len(recipients) != 1:
raise ValueError(
f'All missed_messages must have the same recipient and topic {recipients!r}',
)
# This link is no longer a part of the email, but keeping the code in case
# we find a clean way to add it back in the future
unsubscribe_link = one_click_unsubscribe_link(user_profile,
"missed_messages")
context = common_context(user_profile)
context.update({
'name':
user_profile.full_name,
'message_count':
message_count,
'unsubscribe_link':
unsubscribe_link,
'realm_name_in_notifications':
user_profile.realm_name_in_notifications,
})
triggers = list(message['trigger'] for message in missed_messages)
unique_triggers = set(triggers)
context.update({
'mention':
'mentioned' in unique_triggers
or 'wildcard_mentioned' in unique_triggers,
'stream_email_notify':
'stream_email_notify' in unique_triggers,
'mention_count':
triggers.count('mentioned') + triggers.count("wildcard_mentioned"),
})
# If this setting (email mirroring integration) is enabled, only then
# can users reply to email to send message to Zulip. Thus, one must
# ensure to display warning in the template.
if settings.EMAIL_GATEWAY_PATTERN:
context.update({
'reply_to_zulip': True,
})
else:
context.update({
'reply_to_zulip': False,
})
from zerver.lib.email_mirror import create_missed_message_address
reply_to_address = create_missed_message_address(
user_profile, missed_messages[0]['message'])
if reply_to_address == FromAddress.NOREPLY:
reply_to_name = ""
else:
reply_to_name = "925"
narrow_url = get_narrow_url(user_profile, missed_messages[0]['message'])
context.update({
'narrow_url': narrow_url,
})
senders = list({m['message'].sender for m in missed_messages})
if (missed_messages[0]['message'].recipient.type == Recipient.HUDDLE):
display_recipient = get_display_recipient(
missed_messages[0]['message'].recipient)
# Make sure that this is a list of strings, not a string.
assert not isinstance(display_recipient, str)
other_recipients = [
r['full_name'] for r in display_recipient
if r['id'] != user_profile.id
]
context.update({'group_pm': True})
if len(other_recipients) == 2:
huddle_display_name = " and ".join(other_recipients)
context.update({'huddle_display_name': huddle_display_name})
elif len(other_recipients) == 3:
huddle_display_name = f"{other_recipients[0]}, {other_recipients[1]}, and {other_recipients[2]}"
context.update({'huddle_display_name': huddle_display_name})
else:
huddle_display_name = "{}, and {} others".format(
', '.join(other_recipients[:2]),
len(other_recipients) - 2)
context.update({'huddle_display_name': huddle_display_name})
elif (missed_messages[0]['message'].recipient.type == Recipient.PERSONAL):
context.update({'private_message': True})
elif (context['mention'] or context['stream_email_notify']):
# Keep only the senders who actually mentioned the user
if context['mention']:
senders = list({
m['message'].sender
for m in missed_messages if m['trigger'] == 'mentioned'
or m['trigger'] == 'wildcard_mentioned'
})
message = missed_messages[0]['message']
stream = Stream.objects.only('id',
'name').get(id=message.recipient.type_id)
stream_header = f"{stream.name} > {message.topic_name()}"
context.update({
'stream_header': stream_header,
})
else:
raise AssertionError("Invalid messages!")
# If message content is disabled, then flush all information we pass to email.
if not message_content_allowed_in_missedmessage_emails(user_profile):
realm = user_profile.realm
context.update({
'reply_to_zulip':
False,
'messages': [],
'sender_str':
"",
'realm_str':
realm.name,
'huddle_display_name':
"",
'show_message_content':
False,
'message_content_disabled_by_user':
not user_profile.message_content_in_email_notifications,
'message_content_disabled_by_realm':
not realm.message_content_allowed_in_email_notifications,
})
else:
context.update({
'messages':
build_message_list(user_profile,
list(m['message'] for m in missed_messages)),
'sender_str':
", ".join(sender.full_name for sender in senders),
'realm_str':
user_profile.realm.name,
'show_message_content':
True,
})
with override_language(user_profile.default_language):
from_name: str = _("925 Missed Messages")
from_address = FromAddress.NOREPLY
if len(senders) == 1 and settings.SEND_MISSED_MESSAGE_EMAILS_AS_USER:
# If this setting is enabled, you can reply to the Zulip
# missed message emails directly back to the original sender.
# However, one must ensure the Zulip server is in the SPF
# record for the domain, or there will be spam/deliverability
# problems.
#
# Also, this setting is not really compatible with
# EMAIL_ADDRESS_VISIBILITY_ADMINS.
sender = senders[0]
from_name, from_address = (sender.full_name, sender.email)
context.update({
'reply_to_zulip': False,
})
email_dict = {
'template_prefix':
'zerver/emails/missed_message',
'to_user_ids': [user_profile.id],
'from_name':
from_name,
'from_address':
from_address,
'reply_to_email':
str(Address(display_name=reply_to_name, addr_spec=reply_to_address)),
'context':
context
}
queue_json_publish("email_senders", email_dict)
user_profile.last_reminder = timezone_now()
user_profile.save(update_fields=['last_reminder'])
def list_to_streams(streams_raw: Iterable[Mapping[str, Any]],
user_profile: UserProfile,
autocreate: bool=False) -> Tuple[List[Stream], List[Stream]]:
"""Converts list of dicts to a list of Streams, validating input in the process
For each stream name, we validate it to ensure it meets our
requirements for a proper stream name using check_stream_name.
This function in autocreate mode should be atomic: either an exception will be raised
during a precheck, or all the streams specified will have been created if applicable.
@param streams_raw The list of stream dictionaries to process;
names should already be stripped of whitespace by the caller.
@param user_profile The user for whom we are retreiving the streams
@param autocreate Whether we should create streams if they don't already exist
"""
# Validate all streams, getting extant ones, then get-or-creating the rest.
stream_set = set(stream_dict["name"] for stream_dict in streams_raw)
for stream_name in stream_set:
# Stream names should already have been stripped by the
# caller, but it makes sense to verify anyway.
assert stream_name == stream_name.strip()
check_stream_name(stream_name)
existing_streams = [] # type: List[Stream]
missing_stream_dicts = [] # type: List[Mapping[str, Any]]
existing_stream_map = bulk_get_streams(user_profile.realm, stream_set)
member_creating_announcement_only_stream = False
for stream_dict in streams_raw:
stream_name = stream_dict["name"]
stream = existing_stream_map.get(stream_name.lower())
if stream is None:
# Non admins cannot create STREAM_POST_POLICY_ADMINS streams.
if ((stream_dict.get("stream_post_policy", False) ==
Stream.STREAM_POST_POLICY_ADMINS) and not user_profile.is_realm_admin):
member_creating_announcement_only_stream = True
# New members cannot create STREAM_POST_POLICY_RESTRICT_NEW_MEMBERS streams,
# unless they are admins who are also new members of the organization.
if ((stream_dict.get("stream_post_policy", False) ==
Stream.STREAM_POST_POLICY_RESTRICT_NEW_MEMBERS) and user_profile.is_new_member):
if not user_profile.is_realm_admin:
member_creating_announcement_only_stream = True
missing_stream_dicts.append(stream_dict)
else:
existing_streams.append(stream)
if len(missing_stream_dicts) == 0:
# This is the happy path for callers who expected all of these
# streams to exist already.
created_streams = [] # type: List[Stream]
else:
# autocreate=True path starts here
if not user_profile.can_create_streams():
raise JsonableError(_('User cannot create streams.'))
elif not autocreate:
raise JsonableError(_("Stream(s) (%s) do not exist") % ", ".join(
stream_dict["name"] for stream_dict in missing_stream_dicts))
elif member_creating_announcement_only_stream:
raise JsonableError(_('User cannot create a stream with these settings.'))
# We already filtered out existing streams, so dup_streams
# will normally be an empty list below, but we protect against somebody
# else racing to create the same stream. (This is not an entirely
# paranoid approach, since often on Zulip two people will discuss
# creating a new stream, and both people eagerly do it.)
created_streams, dup_streams = create_streams_if_needed(realm=user_profile.realm,
stream_dicts=missing_stream_dicts)
existing_streams += dup_streams
return existing_streams, created_streams
def do_import_realm(import_dir: Path, subdomain: str) -> Realm:
logging.info("Importing realm dump %s" % (import_dir, ))
if not os.path.exists(import_dir):
raise Exception("Missing import directory!")
realm_data_filename = os.path.join(import_dir, "realm.json")
if not os.path.exists(realm_data_filename):
raise Exception("Missing realm.json file!")
logging.info("Importing realm data from %s" % (realm_data_filename, ))
with open(realm_data_filename) as f:
data = ujson.load(f)
update_model_ids(Stream, data, 'zerver_stream', 'stream')
re_map_foreign_keys(data,
'zerver_realm',
'notifications_stream',
related_table="stream")
fix_datetime_fields(data, 'zerver_realm')
# Fix realm subdomain information
data['zerver_realm'][0]['string_id'] = subdomain
data['zerver_realm'][0]['name'] = subdomain
fix_realm_authentication_bitfield(data, 'zerver_realm',
'authentication_methods')
update_model_ids(Realm, data, 'zerver_realm', 'realm')
realm = Realm(**data['zerver_realm'][0])
if realm.notifications_stream_id is not None:
notifications_stream_id = int(
realm.notifications_stream_id) # type: Optional[int]
else:
notifications_stream_id = None
realm.notifications_stream_id = None
realm.save()
bulk_import_client(data, Client, 'zerver_client')
# Email tokens will automatically be randomly generated when the
# Stream objects are created by Django.
fix_datetime_fields(data, 'zerver_stream')
re_map_foreign_keys(data, 'zerver_stream', 'realm', related_table="realm")
bulk_import_model(data, Stream, 'zerver_stream')
realm.notifications_stream_id = notifications_stream_id
realm.save()
# Remap the user IDs for notification_bot and friends to their
# appropriate IDs on this server
for item in data['zerver_userprofile_crossrealm']:
logging.info("Adding to ID map: %s %s" %
(item['id'], get_system_bot(item['email']).id))
new_user_id = get_system_bot(item['email']).id
update_id_map(table='user_profile',
old_id=item['id'],
new_id=new_user_id)
# Merge in zerver_userprofile_mirrordummy
data['zerver_userprofile'] = data['zerver_userprofile'] + data[
'zerver_userprofile_mirrordummy']
del data['zerver_userprofile_mirrordummy']
data['zerver_userprofile'].sort(key=lambda r: r['id'])
# To remap foreign key for UserProfile.last_active_message_id
update_message_foreign_keys(import_dir)
fix_datetime_fields(data, 'zerver_userprofile')
update_model_ids(UserProfile, data, 'zerver_userprofile', 'user_profile')
re_map_foreign_keys(data,
'zerver_userprofile',
'realm',
related_table="realm")
re_map_foreign_keys(data,
'zerver_userprofile',
'bot_owner',
related_table="user_profile")
re_map_foreign_keys(data,
'zerver_userprofile',
'default_sending_stream',
related_table="stream")
re_map_foreign_keys(data,
'zerver_userprofile',
'default_events_register_stream',
related_table="stream")
re_map_foreign_keys(data,
'zerver_userprofile',
'last_active_message_id',
related_table="message",
id_field=True)
for user_profile_dict in data['zerver_userprofile']:
user_profile_dict['password'] = None
user_profile_dict['api_key'] = random_api_key()
# Since Zulip doesn't use these permissions, drop them
del user_profile_dict['user_permissions']
del user_profile_dict['groups']
user_profiles = [
UserProfile(**item) for item in data['zerver_userprofile']
]
for user_profile in user_profiles:
user_profile.set_unusable_password()
UserProfile.objects.bulk_create(user_profiles)
re_map_foreign_keys(data,
'zerver_defaultstream',
'stream',
related_table="stream")
re_map_foreign_keys(data,
'zerver_realmemoji',
'author',
related_table="user_profile")
for (table, model, related_table) in realm_tables:
re_map_foreign_keys(data, table, 'realm', related_table="realm")
update_model_ids(model, data, table, related_table)
bulk_import_model(data, model, table)
if 'zerver_huddle' in data:
update_model_ids(Huddle, data, 'zerver_huddle', 'huddle')
re_map_foreign_keys(data,
'zerver_recipient',
'type_id',
related_table="stream",
recipient_field=True,
id_field=True)
re_map_foreign_keys(data,
'zerver_recipient',
'type_id',
related_table="user_profile",
recipient_field=True,
id_field=True)
re_map_foreign_keys(data,
'zerver_recipient',
'type_id',
related_table="huddle",
recipient_field=True,
id_field=True)
update_model_ids(Recipient, data, 'zerver_recipient', 'recipient')
bulk_import_model(data, Recipient, 'zerver_recipient')
re_map_foreign_keys(data,
'zerver_subscription',
'user_profile',
related_table="user_profile")
get_huddles_from_subscription(data, 'zerver_subscription')
re_map_foreign_keys(data,
'zerver_subscription',
'recipient',
related_table="recipient")
update_model_ids(Subscription, data, 'zerver_subscription', 'subscription')
bulk_import_model(data, Subscription, 'zerver_subscription')
if 'zerver_realmauditlog' in data:
fix_datetime_fields(data, 'zerver_realmauditlog')
re_map_foreign_keys(data,
'zerver_realmauditlog',
'realm',
related_table="realm")
re_map_foreign_keys(data,
'zerver_realmauditlog',
'modified_user',
related_table='user_profile')
re_map_foreign_keys(data,
'zerver_realmauditlog',
'acting_user',
related_table='user_profile')
re_map_foreign_keys(data,
'zerver_realmauditlog',
'modified_stream',
related_table="stream")
update_model_ids(RealmAuditLog,
data,
'zerver_realmauditlog',
related_table="realmauditlog")
bulk_import_model(data, RealmAuditLog, 'zerver_realmauditlog')
else:
create_subscription_events(data, 'zerver_subscription')
if 'zerver_huddle' in data:
process_huddle_hash(data, 'zerver_huddle')
bulk_import_model(data, Huddle, 'zerver_huddle')
fix_datetime_fields(data, 'zerver_userpresence')
re_map_foreign_keys(data,
'zerver_userpresence',
'user_profile',
related_table="user_profile")
re_map_foreign_keys(data,
'zerver_userpresence',
'client',
related_table='client')
update_model_ids(UserPresence, data, 'zerver_userpresence',
'user_presence')
bulk_import_model(data, UserPresence, 'zerver_userpresence')
fix_datetime_fields(data, 'zerver_useractivity')
re_map_foreign_keys(data,
'zerver_useractivity',
'user_profile',
related_table="user_profile")
re_map_foreign_keys(data,
'zerver_useractivity',
'client',
related_table='client')
update_model_ids(UserActivity, data, 'zerver_useractivity', 'useractivity')
bulk_import_model(data, UserActivity, 'zerver_useractivity')
fix_datetime_fields(data, 'zerver_useractivityinterval')
re_map_foreign_keys(data,
'zerver_useractivityinterval',
'user_profile',
related_table="user_profile")
update_model_ids(UserActivityInterval, data, 'zerver_useractivityinterval',
'useractivityinterval')
bulk_import_model(data, UserActivityInterval,
'zerver_useractivityinterval')
re_map_foreign_keys(data,
'zerver_customprofilefield',
'realm',
related_table="realm")
update_model_ids(CustomProfileField,
data,
'zerver_customprofilefield',
related_table="customprofilefield")
bulk_import_model(data, CustomProfileField, 'zerver_customprofilefield')
re_map_foreign_keys(data,
'zerver_customprofilefieldvalue',
'user_profile',
related_table="user_profile")
re_map_foreign_keys(data,
'zerver_customprofilefieldvalue',
'field',
related_table="customprofilefield")
update_model_ids(CustomProfileFieldValue,
data,
'zerver_customprofilefieldvalue',
related_table="customprofilefieldvalue")
bulk_import_model(data, CustomProfileFieldValue,
'zerver_customprofilefieldvalue')
# Import uploaded files and avatars
import_uploads(os.path.join(import_dir, "avatars"),
processing_avatars=True)
import_uploads(os.path.join(import_dir, "uploads"))
# We need to have this check as the emoji files are only present in the data
# importer from slack
# For Zulip export, this doesn't exist
if os.path.exists(os.path.join(import_dir, "emoji")):
import_uploads(os.path.join(import_dir, "emoji"),
processing_emojis=True)
# Import zerver_message and zerver_usermessage
import_message_data(import_dir)
re_map_foreign_keys(data,
'zerver_reaction',
'message',
related_table="message")
re_map_foreign_keys(data,
'zerver_reaction',
'user_profile',
related_table="user_profile")
re_map_foreign_keys(data,
'zerver_reaction',
'emoji_code',
related_table="realmemoji",
id_field=True,
reaction_field=True)
update_model_ids(Reaction, data, 'zerver_reaction', 'reaction')
bulk_import_model(data, Reaction, 'zerver_reaction')
# Do attachments AFTER message data is loaded.
# TODO: de-dup how we read these json files.
fn = os.path.join(import_dir, "attachment.json")
if not os.path.exists(fn):
raise Exception("Missing attachment.json file!")
logging.info("Importing attachment data from %s" % (fn, ))
with open(fn) as f:
data = ujson.load(f)
import_attachments(data)
return realm
def login_and_go_to_home(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
# Mark the user as having been just created, so no "new login" email is sent
user_profile.just_registered = True
do_login(request, user_profile)
return HttpResponseRedirect(user_profile.realm.uri + reverse('zerver.views.home.home'))
def do_import_realm(import_dir: Path, subdomain: str) -> Realm:
logging.info("Importing realm dump %s" % (import_dir,))
if not os.path.exists(import_dir):
raise Exception("Missing import directory!")
realm_data_filename = os.path.join(import_dir, "realm.json")
if not os.path.exists(realm_data_filename):
raise Exception("Missing realm.json file!")
logging.info("Importing realm data from %s" % (realm_data_filename,))
with open(realm_data_filename) as f:
data = ujson.load(f)
sort_by_date = data.get('sort_by_date', False)
bulk_import_client(data, Client, 'zerver_client')
# We don't import the Stream model yet, since it depends on Realm,
# which isn't imported yet. But we need the Stream model IDs for
# notifications_stream.
update_model_ids(Stream, data, 'stream')
re_map_foreign_keys(data, 'zerver_realm', 'notifications_stream', related_table="stream")
re_map_foreign_keys(data, 'zerver_realm', 'signup_notifications_stream', related_table="stream")
fix_datetime_fields(data, 'zerver_realm')
# Fix realm subdomain information
data['zerver_realm'][0]['string_id'] = subdomain
data['zerver_realm'][0]['name'] = subdomain
fix_realm_authentication_bitfield(data, 'zerver_realm', 'authentication_methods')
update_model_ids(Realm, data, 'realm')
realm = Realm(**data['zerver_realm'][0])
if realm.notifications_stream_id is not None:
notifications_stream_id = int(realm.notifications_stream_id) # type: Optional[int]
else:
notifications_stream_id = None
realm.notifications_stream_id = None
if realm.signup_notifications_stream_id is not None:
signup_notifications_stream_id = int(realm.signup_notifications_stream_id) # type: Optional[int]
else:
signup_notifications_stream_id = None
realm.signup_notifications_stream_id = None
realm.save()
# Email tokens will automatically be randomly generated when the
# Stream objects are created by Django.
fix_datetime_fields(data, 'zerver_stream')
re_map_foreign_keys(data, 'zerver_stream', 'realm', related_table="realm")
bulk_import_model(data, Stream)
realm.notifications_stream_id = notifications_stream_id
realm.signup_notifications_stream_id = signup_notifications_stream_id
realm.save()
# Remap the user IDs for notification_bot and friends to their
# appropriate IDs on this server
for item in data['zerver_userprofile_crossrealm']:
logging.info("Adding to ID map: %s %s" % (item['id'], get_system_bot(item['email']).id))
new_user_id = get_system_bot(item['email']).id
update_id_map(table='user_profile', old_id=item['id'], new_id=new_user_id)
new_recipient_id = Recipient.objects.get(type=Recipient.PERSONAL, type_id=new_user_id).id
update_id_map(table='recipient', old_id=item['recipient_id'], new_id=new_recipient_id)
# Merge in zerver_userprofile_mirrordummy
data['zerver_userprofile'] = data['zerver_userprofile'] + data['zerver_userprofile_mirrordummy']
del data['zerver_userprofile_mirrordummy']
data['zerver_userprofile'].sort(key=lambda r: r['id'])
# To remap foreign key for UserProfile.last_active_message_id
update_message_foreign_keys(import_dir=import_dir, sort_by_date=sort_by_date)
fix_datetime_fields(data, 'zerver_userprofile')
update_model_ids(UserProfile, data, 'user_profile')
re_map_foreign_keys(data, 'zerver_userprofile', 'realm', related_table="realm")
re_map_foreign_keys(data, 'zerver_userprofile', 'bot_owner', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_userprofile', 'default_sending_stream',
related_table="stream")
re_map_foreign_keys(data, 'zerver_userprofile', 'default_events_register_stream',
related_table="stream")
re_map_foreign_keys(data, 'zerver_userprofile', 'last_active_message_id',
related_table="message", id_field=True)
for user_profile_dict in data['zerver_userprofile']:
user_profile_dict['password'] = None
user_profile_dict['api_key'] = generate_api_key()
# Since Zulip doesn't use these permissions, drop them
del user_profile_dict['user_permissions']
del user_profile_dict['groups']
user_profiles = [UserProfile(**item) for item in data['zerver_userprofile']]
for user_profile in user_profiles:
user_profile.set_unusable_password()
UserProfile.objects.bulk_create(user_profiles)
re_map_foreign_keys(data, 'zerver_defaultstream', 'stream', related_table="stream")
re_map_foreign_keys(data, 'zerver_realmemoji', 'author', related_table="user_profile")
for (table, model, related_table) in realm_tables:
re_map_foreign_keys(data, table, 'realm', related_table="realm")
update_model_ids(model, data, related_table)
bulk_import_model(data, model)
if 'zerver_huddle' in data:
update_model_ids(Huddle, data, 'huddle')
# We don't import Huddle yet, since we don't have the data to
# compute huddle hashes until we've imported some of the
# tables below.
# TODO: double-check this.
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="stream",
recipient_field=True, id_field=True)
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="user_profile",
recipient_field=True, id_field=True)
re_map_foreign_keys(data, 'zerver_recipient', 'type_id', related_table="huddle",
recipient_field=True, id_field=True)
update_model_ids(Recipient, data, 'recipient')
bulk_import_model(data, Recipient)
re_map_foreign_keys(data, 'zerver_subscription', 'user_profile', related_table="user_profile")
get_huddles_from_subscription(data, 'zerver_subscription')
re_map_foreign_keys(data, 'zerver_subscription', 'recipient', related_table="recipient")
update_model_ids(Subscription, data, 'subscription')
bulk_import_model(data, Subscription)
if 'zerver_realmauditlog' in data:
fix_datetime_fields(data, 'zerver_realmauditlog')
re_map_foreign_keys(data, 'zerver_realmauditlog', 'realm', related_table="realm")
re_map_foreign_keys(data, 'zerver_realmauditlog', 'modified_user',
related_table='user_profile')
re_map_foreign_keys(data, 'zerver_realmauditlog', 'acting_user',
related_table='user_profile')
re_map_foreign_keys(data, 'zerver_realmauditlog', 'modified_stream',
related_table="stream")
update_model_ids(RealmAuditLog, data, related_table="realmauditlog")
bulk_import_model(data, RealmAuditLog)
else:
logging.info('about to call create_subscription_events')
create_subscription_events(
data=data,
realm_id=realm.id,
)
logging.info('done with create_subscription_events')
if 'zerver_huddle' in data:
process_huddle_hash(data, 'zerver_huddle')
bulk_import_model(data, Huddle)
if 'zerver_userhotspot' in data:
fix_datetime_fields(data, 'zerver_userhotspot')
re_map_foreign_keys(data, 'zerver_userhotspot', 'user', related_table='user_profile')
update_model_ids(UserHotspot, data, 'userhotspot')
bulk_import_model(data, UserHotspot)
if 'zerver_mutedtopic' in data:
re_map_foreign_keys(data, 'zerver_mutedtopic', 'user_profile', related_table='user_profile')
re_map_foreign_keys(data, 'zerver_mutedtopic', 'stream', related_table='stream')
re_map_foreign_keys(data, 'zerver_mutedtopic', 'recipient', related_table='recipient')
update_model_ids(MutedTopic, data, 'mutedtopic')
bulk_import_model(data, MutedTopic)
if 'zerver_service' in data:
re_map_foreign_keys(data, 'zerver_service', 'user_profile', related_table='user_profile')
fix_service_tokens(data, 'zerver_service')
update_model_ids(Service, data, 'service')
bulk_import_model(data, Service)
if 'zerver_usergroup' in data:
re_map_foreign_keys(data, 'zerver_usergroup', 'realm', related_table='realm')
re_map_foreign_keys_many_to_many(data, 'zerver_usergroup',
'members', related_table='user_profile')
update_model_ids(UserGroup, data, 'usergroup')
bulk_import_model(data, UserGroup)
re_map_foreign_keys(data, 'zerver_usergroupmembership',
'user_group', related_table='usergroup')
re_map_foreign_keys(data, 'zerver_usergroupmembership',
'user_profile', related_table='user_profile')
update_model_ids(UserGroupMembership, data, 'usergroupmembership')
bulk_import_model(data, UserGroupMembership)
if 'zerver_botstoragedata' in data:
re_map_foreign_keys(data, 'zerver_botstoragedata', 'bot_profile', related_table='user_profile')
update_model_ids(BotStorageData, data, 'botstoragedata')
bulk_import_model(data, BotStorageData)
if 'zerver_botconfigdata' in data:
re_map_foreign_keys(data, 'zerver_botconfigdata', 'bot_profile', related_table='user_profile')
update_model_ids(BotConfigData, data, 'botconfigdata')
bulk_import_model(data, BotConfigData)
fix_datetime_fields(data, 'zerver_userpresence')
re_map_foreign_keys(data, 'zerver_userpresence', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_userpresence', 'client', related_table='client')
update_model_ids(UserPresence, data, 'user_presence')
bulk_import_model(data, UserPresence)
fix_datetime_fields(data, 'zerver_useractivity')
re_map_foreign_keys(data, 'zerver_useractivity', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_useractivity', 'client', related_table='client')
update_model_ids(UserActivity, data, 'useractivity')
bulk_import_model(data, UserActivity)
fix_datetime_fields(data, 'zerver_useractivityinterval')
re_map_foreign_keys(data, 'zerver_useractivityinterval', 'user_profile', related_table="user_profile")
update_model_ids(UserActivityInterval, data, 'useractivityinterval')
bulk_import_model(data, UserActivityInterval)
re_map_foreign_keys(data, 'zerver_customprofilefield', 'realm', related_table="realm")
update_model_ids(CustomProfileField, data, related_table="customprofilefield")
bulk_import_model(data, CustomProfileField)
re_map_foreign_keys(data, 'zerver_customprofilefieldvalue', 'user_profile',
related_table="user_profile")
re_map_foreign_keys(data, 'zerver_customprofilefieldvalue', 'field',
related_table="customprofilefield")
fix_customprofilefield(data)
update_model_ids(CustomProfileFieldValue, data, related_table="customprofilefieldvalue")
bulk_import_model(data, CustomProfileFieldValue)
# Import uploaded files and avatars
import_uploads(os.path.join(import_dir, "avatars"), processing_avatars=True)
import_uploads(os.path.join(import_dir, "uploads"))
# We need to have this check as the emoji files are only present in the data
# importer from slack
# For Zulip export, this doesn't exist
if os.path.exists(os.path.join(import_dir, "emoji")):
import_uploads(os.path.join(import_dir, "emoji"), processing_emojis=True)
sender_map = {
user['id']: user
for user in data['zerver_userprofile']
}
# Import zerver_message and zerver_usermessage
import_message_data(realm=realm, sender_map=sender_map, import_dir=import_dir)
re_map_foreign_keys(data, 'zerver_reaction', 'message', related_table="message")
re_map_foreign_keys(data, 'zerver_reaction', 'user_profile', related_table="user_profile")
re_map_foreign_keys(data, 'zerver_reaction', 'emoji_code', related_table="realmemoji", id_field=True,
reaction_field=True)
update_model_ids(Reaction, data, 'reaction')
bulk_import_model(data, Reaction)
# Do attachments AFTER message data is loaded.
# TODO: de-dup how we read these json files.
fn = os.path.join(import_dir, "attachment.json")
if not os.path.exists(fn):
raise Exception("Missing attachment.json file!")
logging.info("Importing attachment data from %s" % (fn,))
with open(fn) as f:
data = ujson.load(f)
import_attachments(data)
if settings.BILLING_ENABLED:
do_change_plan_type(realm, Realm.LIMITED)
return realm
def validation_func(user_profile: UserProfile) -> bool:
user_profile.refresh_from_db()
return user_profile.can_edit_user_groups()
def add_subscriptions_backend(
request: HttpRequest,
user_profile: UserProfile,
streams_raw: Iterable[Dict[str, str]] = REQ(
"subscriptions",
validator=check_list(
check_dict_only([('name', check_string)],
optional_keys=[
('color', check_color),
('description',
check_capped_string(
Stream.MAX_DESCRIPTION_LENGTH)),
]))),
invite_only: bool = REQ(validator=check_bool, default=False),
stream_post_policy: int = REQ(validator=check_int_in(
Stream.STREAM_POST_POLICY_TYPES),
default=Stream.STREAM_POST_POLICY_EVERYONE),
history_public_to_subscribers: Optional[bool] = REQ(validator=check_bool,
default=None),
announce: bool = REQ(validator=check_bool, default=False),
principals: List[str] = REQ(validator=check_list(check_string),
default=[]),
authorization_errors_fatal: bool = REQ(validator=check_bool, default=True),
) -> HttpResponse:
stream_dicts = []
color_map = {}
for stream_dict in streams_raw:
# 'color' field is optional
# check for its presence in the streams_raw first
if 'color' in stream_dict:
color_map[stream_dict['name']] = stream_dict['color']
if 'description' in stream_dict:
# We don't allow newline characters in stream descriptions.
stream_dict['description'] = stream_dict['description'].replace(
"\n", " ")
stream_dict_copy = {} # type: Dict[str, Any]
for field in stream_dict:
stream_dict_copy[field] = stream_dict[field]
# Strip the stream name here.
stream_dict_copy['name'] = stream_dict_copy['name'].strip()
stream_dict_copy["invite_only"] = invite_only
stream_dict_copy["stream_post_policy"] = stream_post_policy
stream_dict_copy[
"history_public_to_subscribers"] = history_public_to_subscribers
stream_dicts.append(stream_dict_copy)
# Validation of the streams arguments, including enforcement of
# can_create_streams policy and check_stream_name policy is inside
# list_to_streams.
existing_streams, created_streams = \
list_to_streams(stream_dicts, user_profile, autocreate=True)
authorized_streams, unauthorized_streams = \
filter_stream_authorization(user_profile, existing_streams)
if len(unauthorized_streams) > 0 and authorization_errors_fatal:
return json_error(
_("Unable to access stream (%s).") % unauthorized_streams[0].name)
# Newly created streams are also authorized for the creator
streams = authorized_streams + created_streams
if len(principals) > 0:
if user_profile.realm.is_zephyr_mirror_realm and not all(
stream.invite_only for stream in streams):
return json_error(
_("You can only invite other Zephyr mirroring users to private streams."
))
if not user_profile.can_subscribe_other_users():
if user_profile.realm.invite_to_stream_policy == Realm.INVITE_TO_STREAM_POLICY_ADMINS:
return json_error(
_("Only administrators can modify other users' subscriptions."
))
# Realm.INVITE_TO_STREAM_POLICY_MEMBERS only fails if the
# user is a guest, which happens in the decorator above.
assert user_profile.realm.invite_to_stream_policy == \
Realm.INVITE_TO_STREAM_POLICY_WAITING_PERIOD
return json_error(
_("Your account is too new to modify other users' subscriptions."
))
subscribers = set(
principal_to_user_profile(user_profile, principal)
for principal in principals)
else:
subscribers = set([user_profile])
(subscribed,
already_subscribed) = bulk_add_subscriptions(streams,
subscribers,
acting_user=user_profile,
color_map=color_map)
# We can assume unique emails here for now, but we should eventually
# convert this function to be more id-centric.
email_to_user_profile = dict() # type: Dict[str, UserProfile]
result = dict(subscribed=defaultdict(list),
already_subscribed=defaultdict(list)) # type: Dict[str, Any]
for (subscriber, stream) in subscribed:
result["subscribed"][subscriber.email].append(stream.name)
email_to_user_profile[subscriber.email] = subscriber
for (subscriber, stream) in already_subscribed:
result["already_subscribed"][subscriber.email].append(stream.name)
bots = dict(
(subscriber.email, subscriber.is_bot) for subscriber in subscribers)
newly_created_stream_names = {s.name for s in created_streams}
# Inform the user if someone else subscribed them to stuff,
# or if a new stream was created with the "announce" option.
notifications = []
if len(principals) > 0 and result["subscribed"]:
for email, subscribed_stream_names in result["subscribed"].items():
if email == user_profile.email:
# Don't send a Zulip if you invited yourself.
continue
if bots[email]:
# Don't send invitation Zulips to bots
continue
# For each user, we notify them about newly subscribed streams, except for
# streams that were newly created.
notify_stream_names = set(
subscribed_stream_names) - newly_created_stream_names
if not notify_stream_names:
continue
msg = you_were_just_subscribed_message(
acting_user=user_profile,
stream_names=notify_stream_names,
)
sender = get_system_bot(settings.NOTIFICATION_BOT)
notifications.append(
internal_prep_private_message(
realm=user_profile.realm,
sender=sender,
recipient_user=email_to_user_profile[email],
content=msg))
if announce and len(created_streams) > 0:
notifications_stream = user_profile.realm.get_notifications_stream()
if notifications_stream is not None:
if len(created_streams) > 1:
content = _(
"@_**%(user_name)s|%(user_id)d** created the following streams: %(stream_str)s."
)
else:
content = _(
"@_**%(user_name)s|%(user_id)d** created a new stream %(stream_str)s."
)
content = content % {
'user_name':
user_profile.full_name,
'user_id':
user_profile.id,
'stream_str':
", ".join('#**%s**' % (s.name, ) for s in created_streams)
}
sender = get_system_bot(settings.NOTIFICATION_BOT)
topic = _('new streams')
notifications.append(
internal_prep_stream_message(
realm=user_profile.realm,
sender=sender,
stream=notifications_stream,
topic=topic,
content=content,
))
if not user_profile.realm.is_zephyr_mirror_realm and len(
created_streams) > 0:
sender = get_system_bot(settings.NOTIFICATION_BOT)
for stream in created_streams:
notifications.append(
internal_prep_stream_message(
realm=user_profile.realm,
sender=sender,
stream=stream,
topic=Realm.STREAM_EVENTS_NOTIFICATION_TOPIC,
content=_(
'Stream created by @_**%(user_name)s|%(user_id)d**.') %
{
'user_name': user_profile.full_name,
'user_id': user_profile.id
}))
if len(notifications) > 0:
do_send_messages(notifications, mark_as_read=[user_profile.id])
result["subscribed"] = dict(result["subscribed"])
result["already_subscribed"] = dict(result["already_subscribed"])
if not authorization_errors_fatal:
result["unauthorized"] = [s.name for s in unauthorized_streams]
return json_success(result)
def fetch_initial_state_data(user_profile: UserProfile,
event_types: Optional[Iterable[str]],
queue_id: str, client_gravatar: bool,
include_subscribers: bool = True) -> Dict[str, Any]:
state = {'queue_id': queue_id} # type: Dict[str, Any]
realm = user_profile.realm
if event_types is None:
# return True always
want = always_want # type: Callable[[str], bool]
else:
want = set(event_types).__contains__
if want('alert_words'):
state['alert_words'] = user_alert_words(user_profile)
if want('custom_profile_fields'):
fields = custom_profile_fields_for_realm(realm.id)
state['custom_profile_fields'] = [f.as_dict() for f in fields]
state['custom_profile_field_types'] = CustomProfileField.FIELD_TYPE_CHOICES_DICT
if want('hotspots'):
state['hotspots'] = get_next_hotspots(user_profile)
if want('message'):
# The client should use get_messages() to fetch messages
# starting with the max_message_id. They will get messages
# newer than that ID via get_events()
messages = Message.objects.filter(usermessage__user_profile=user_profile).order_by('-id')[:1]
if messages:
state['max_message_id'] = messages[0].id
else:
state['max_message_id'] = -1
if want('muted_topics'):
state['muted_topics'] = get_topic_mutes(user_profile)
if want('pointer'):
state['pointer'] = user_profile.pointer
if want('presence'):
state['presences'] = get_status_dict(user_profile)
if want('realm'):
for property_name in Realm.property_types:
state['realm_' + property_name] = getattr(realm, property_name)
# Most state is handled via the property_types framework;
# these manual entries are for those realm settings that don't
# fit into that framework.
state['realm_authentication_methods'] = realm.authentication_methods_dict()
state['realm_allow_message_editing'] = realm.allow_message_editing
state['realm_allow_community_topic_editing'] = realm.allow_community_topic_editing
state['realm_allow_message_deleting'] = realm.allow_message_deleting
state['realm_message_content_edit_limit_seconds'] = realm.message_content_edit_limit_seconds
state['realm_message_content_delete_limit_seconds'] = realm.message_content_delete_limit_seconds
state['realm_icon_url'] = realm_icon_url(realm)
state['realm_icon_source'] = realm.icon_source
state['max_icon_file_size'] = settings.MAX_ICON_FILE_SIZE
state['realm_logo_url'] = realm_logo_url(realm)
state['realm_logo_source'] = realm.logo_source
state['max_logo_file_size'] = settings.MAX_LOGO_FILE_SIZE
state['realm_bot_domain'] = realm.get_bot_domain()
state['realm_uri'] = realm.uri
state['realm_available_video_chat_providers'] = realm.VIDEO_CHAT_PROVIDERS
state['realm_presence_disabled'] = realm.presence_disabled
state['realm_digest_emails_enabled'] = realm.digest_emails_enabled and settings.SEND_DIGEST_EMAILS
state['realm_is_zephyr_mirror_realm'] = realm.is_zephyr_mirror_realm
state['realm_email_auth_enabled'] = email_auth_enabled(realm)
state['realm_password_auth_enabled'] = password_auth_enabled(realm)
state['realm_push_notifications_enabled'] = push_notifications_enabled()
if realm.notifications_stream and not realm.notifications_stream.deactivated:
notifications_stream = realm.notifications_stream
state['realm_notifications_stream_id'] = notifications_stream.id
else:
state['realm_notifications_stream_id'] = -1
signup_notifications_stream = realm.get_signup_notifications_stream()
if signup_notifications_stream:
state['realm_signup_notifications_stream_id'] = signup_notifications_stream.id
else:
state['realm_signup_notifications_stream_id'] = -1
if want('realm_domains'):
state['realm_domains'] = get_realm_domains(realm)
if want('realm_emoji'):
state['realm_emoji'] = realm.get_emoji()
if want('realm_filters'):
state['realm_filters'] = realm_filters_for_realm(realm.id)
if want('realm_user_groups'):
state['realm_user_groups'] = user_groups_in_realm_serialized(realm)
if want('realm_user'):
state['raw_users'] = get_raw_user_data(
realm_id=realm.id,
client_gravatar=client_gravatar,
)
# For the user's own avatar URL, we force
# client_gravatar=False, since that saves some unnecessary
# client-side code for handing medium-size avatars. See #8253
# for details.
state['avatar_source'] = user_profile.avatar_source
state['avatar_url_medium'] = avatar_url(
user_profile,
medium=True,
client_gravatar=False,
)
state['avatar_url'] = avatar_url(
user_profile,
medium=False,
client_gravatar=False,
)
state['can_create_streams'] = user_profile.can_create_streams()
state['can_subscribe_other_users'] = user_profile.can_subscribe_other_users()
state['cross_realm_bots'] = list(get_cross_realm_dicts())
state['is_admin'] = user_profile.is_realm_admin
state['is_guest'] = user_profile.is_guest
state['user_id'] = user_profile.id
state['enter_sends'] = user_profile.enter_sends
state['email'] = user_profile.email
state['delivery_email'] = user_profile.delivery_email
state['full_name'] = user_profile.full_name
if want('realm_bot'):
state['realm_bots'] = get_owned_bot_dicts(user_profile)
# This does not yet have an apply_event counterpart, since currently,
# new entries for EMBEDDED_BOTS can only be added directly in the codebase.
if want('realm_embedded_bots'):
realm_embedded_bots = []
for bot in EMBEDDED_BOTS:
realm_embedded_bots.append({'name': bot.name,
'config': load_bot_config_template(bot.name)})
state['realm_embedded_bots'] = realm_embedded_bots
if want('subscription'):
subscriptions, unsubscribed, never_subscribed = gather_subscriptions_helper(
user_profile, include_subscribers=include_subscribers)
state['subscriptions'] = subscriptions
state['unsubscribed'] = unsubscribed
state['never_subscribed'] = never_subscribed
if want('update_message_flags') and want('message'):
# Keeping unread_msgs updated requires both message flag updates and
# message updates. This is due to the fact that new messages will not
# generate a flag update so we need to use the flags field in the
# message event.
state['raw_unread_msgs'] = get_raw_unread_data(user_profile)
if want('starred_messages'):
state['starred_messages'] = get_starred_message_ids(user_profile)
if want('stream'):
state['streams'] = do_get_streams(user_profile)
state['stream_name_max_length'] = Stream.MAX_NAME_LENGTH
state['stream_description_max_length'] = Stream.MAX_DESCRIPTION_LENGTH
if want('default_streams'):
state['realm_default_streams'] = streams_to_dicts_sorted(
get_default_streams_for_realm(realm.id))
if want('default_stream_groups'):
state['realm_default_stream_groups'] = default_stream_groups_to_dicts_sorted(
get_default_stream_groups(realm))
if want('update_display_settings'):
for prop in UserProfile.property_types:
state[prop] = getattr(user_profile, prop)
state['emojiset_choices'] = user_profile.emojiset_choices()
if want('update_global_notifications'):
for notification in UserProfile.notification_setting_types:
state[notification] = getattr(user_profile, notification)
state['available_notification_sounds'] = get_available_notification_sounds()
if want('user_status'):
state['away_user_ids'] = sorted(list(get_away_user_ids(realm_id=realm.id)))
if want('zulip_version'):
state['zulip_version'] = ZULIP_VERSION
return state
def add_subscriptions_backend(
request: HttpRequest,
user_profile: UserProfile,
streams_raw: Iterable[Mapping[str, str]] = REQ(
"subscriptions",
validator=check_list(check_dict([('name', check_string)]))),
invite_only: bool = REQ(validator=check_bool, default=False),
is_announcement_only: bool = REQ(validator=check_bool, default=False),
history_public_to_subscribers: Optional[bool] = REQ(validator=check_bool,
default=None),
announce: bool = REQ(validator=check_bool, default=False),
principals: List[str] = REQ(validator=check_list(check_string),
default=[]),
authorization_errors_fatal: bool = REQ(validator=check_bool, default=True),
) -> HttpResponse:
stream_dicts = []
for stream_dict in streams_raw:
stream_dict_copy = {} # type: Dict[str, Any]
for field in stream_dict:
stream_dict_copy[field] = stream_dict[field]
# Strip the stream name here.
stream_dict_copy['name'] = stream_dict_copy['name'].strip()
stream_dict_copy["invite_only"] = invite_only
stream_dict_copy["is_announcement_only"] = is_announcement_only
stream_dict_copy[
"history_public_to_subscribers"] = history_public_to_subscribers
stream_dicts.append(stream_dict_copy)
# Validation of the streams arguments, including enforcement of
# can_create_streams policy and check_stream_name policy is inside
# list_to_streams.
existing_streams, created_streams = \
list_to_streams(stream_dicts, user_profile, autocreate=True)
authorized_streams, unauthorized_streams = \
filter_stream_authorization(user_profile, existing_streams)
if len(unauthorized_streams) > 0 and authorization_errors_fatal:
return json_error(
_("Unable to access stream (%s).") % unauthorized_streams[0].name)
# Newly created streams are also authorized for the creator
streams = authorized_streams + created_streams
if len(principals) > 0:
if user_profile.realm.is_zephyr_mirror_realm and not all(
stream.invite_only for stream in streams):
return json_error(
_("You can only invite other Zephyr mirroring users to private streams."
))
if not user_profile.can_subscribe_other_users():
return json_error(
_("Your account is too new to modify other users' subscriptions."
))
subscribers = set(
principal_to_user_profile(user_profile, principal)
for principal in principals)
else:
subscribers = set([user_profile])
(subscribed,
already_subscribed) = bulk_add_subscriptions(streams,
subscribers,
acting_user=user_profile)
# We can assume unique emails here for now, but we should eventually
# convert this function to be more id-centric.
email_to_user_profile = dict() # type: Dict[str, UserProfile]
result = dict(subscribed=defaultdict(list),
already_subscribed=defaultdict(list)) # type: Dict[str, Any]
for (subscriber, stream) in subscribed:
result["subscribed"][subscriber.email].append(stream.name)
email_to_user_profile[subscriber.email] = subscriber
for (subscriber, stream) in already_subscribed:
result["already_subscribed"][subscriber.email].append(stream.name)
bots = dict(
(subscriber.email, subscriber.is_bot) for subscriber in subscribers)
newly_created_stream_names = {s.name for s in created_streams}
# Inform the user if someone else subscribed them to stuff,
# or if a new stream was created with the "announce" option.
notifications = []
if len(principals) > 0 and result["subscribed"]:
for email, subscribed_stream_names in result["subscribed"].items():
if email == user_profile.email:
# Don't send a Zulip if you invited yourself.
continue
if bots[email]:
# Don't send invitation Zulips to bots
continue
# For each user, we notify them about newly subscribed streams, except for
# streams that were newly created.
notify_stream_names = set(
subscribed_stream_names) - newly_created_stream_names
if not notify_stream_names:
continue
msg = you_were_just_subscribed_message(
acting_user=user_profile,
stream_names=notify_stream_names,
)
sender = get_system_bot(settings.NOTIFICATION_BOT)
notifications.append(
internal_prep_private_message(
realm=user_profile.realm,
sender=sender,
recipient_user=email_to_user_profile[email],
content=msg))
if announce and len(
created_streams) > 0 and settings.NOTIFICATION_BOT is not None:
notifications_stream = user_profile.realm.get_notifications_stream()
if notifications_stream is not None:
if len(created_streams) > 1:
stream_strs = ", ".join('#**%s**' % s.name
for s in created_streams)
stream_msg = "the following streams: %s" % (stream_strs, )
else:
stream_msg = "a new stream #**%s**." % created_streams[0].name
msg = ("_@**%s|%d** just created %s" %
(user_profile.full_name, user_profile.id, stream_msg))
sender = get_system_bot(settings.NOTIFICATION_BOT)
stream_name = notifications_stream.name
topic = 'Streams'
notifications.append(
internal_prep_stream_message(realm=user_profile.realm,
sender=sender,
stream_name=stream_name,
topic=topic,
content=msg))
if not user_profile.realm.is_zephyr_mirror_realm:
for stream in created_streams:
notifications.append(prep_stream_welcome_message(stream))
if len(notifications) > 0:
do_send_messages(notifications)
result["subscribed"] = dict(result["subscribed"])
result["already_subscribed"] = dict(result["already_subscribed"])
if not authorization_errors_fatal:
result["unauthorized"] = [s.name for s in unauthorized_streams]
return json_success(result)
def events_register_backend(
request: HttpRequest,
user_profile: UserProfile,
apply_markdown: bool = REQ(default=False, json_validator=check_bool),
client_gravatar: bool = REQ(default=False, json_validator=check_bool),
slim_presence: bool = REQ(default=False, json_validator=check_bool),
all_public_streams: Optional[bool] = REQ(default=None,
json_validator=check_bool),
include_subscribers: bool = REQ(default=False, json_validator=check_bool),
client_capabilities: Optional[Dict[str, bool]] = REQ(
json_validator=check_dict(
[
# This field was accidentally made required when it was added in v2.0.0-781;
# this was not realized until after the release of Zulip 2.1.2. (It remains
# required to help ensure backwards compatibility of client code.)
("notification_settings_null", check_bool),
],
[
# Any new fields of `client_capabilities` should be optional. Add them here.
("bulk_message_deletion", check_bool),
("user_avatar_url_field_optional", check_bool),
],
value_validator=check_bool,
),
default=None,
),
event_types: Optional[Iterable[str]] = REQ(
json_validator=check_list(check_string), default=None),
fetch_event_types: Optional[Iterable[str]] = REQ(
json_validator=check_list(check_string), default=None),
narrow: NarrowT = REQ(json_validator=check_list(
check_list(check_string, length=2)),
default=[]),
queue_lifespan_secs: int = REQ(converter=int,
default=0,
documentation_pending=True),
) -> HttpResponse:
if all_public_streams and not user_profile.can_access_public_streams():
return json_error(_("User not authorized for this query"))
all_public_streams = _default_all_public_streams(user_profile,
all_public_streams)
narrow = _default_narrow(user_profile, narrow)
if client_capabilities is None:
client_capabilities = {}
ret = do_events_register(
user_profile,
request.client,
apply_markdown,
client_gravatar,
slim_presence,
event_types,
queue_lifespan_secs,
all_public_streams,
narrow=narrow,
include_subscribers=include_subscribers,
client_capabilities=client_capabilities,
fetch_event_types=fetch_event_types,
)
return json_success(ret)
def patch_bot_backend(
request: HttpRequest,
user_profile: UserProfile,
email: Text,
full_name: Optional[Text] = REQ(default=None),
bot_owner: Optional[Text] = REQ(default=None),
default_sending_stream: Optional[Text] = REQ(default=None),
default_events_register_stream: Optional[Text] = REQ(default=None),
default_all_public_streams: Optional[bool] = REQ(default=None,
validator=check_bool)
) -> HttpResponse:
try:
bot = get_user(email, user_profile.realm)
except UserProfile.DoesNotExist:
return json_error(_('No such user'))
if not user_profile.can_admin_user(bot):
return json_error(_('Insufficient permission'))
if full_name is not None:
check_change_full_name(bot, full_name, user_profile)
if bot_owner is not None:
owner = get_user(bot_owner, user_profile.realm)
do_change_bot_owner(bot, owner, user_profile)
if default_sending_stream is not None:
if default_sending_stream == "":
stream = None # type: Optional[Stream]
else:
(stream, recipient,
sub) = access_stream_by_name(user_profile, default_sending_stream)
do_change_default_sending_stream(bot, stream)
if default_events_register_stream is not None:
if default_events_register_stream == "":
stream = None
else:
(stream, recipient,
sub) = access_stream_by_name(user_profile,
default_events_register_stream)
do_change_default_events_register_stream(bot, stream)
if default_all_public_streams is not None:
do_change_default_all_public_streams(bot, default_all_public_streams)
if len(request.FILES) == 0:
pass
elif len(request.FILES) == 1:
user_file = list(request.FILES.values())[0]
upload_avatar_image(user_file, user_profile, bot)
avatar_source = UserProfile.AVATAR_FROM_USER
do_change_avatar_fields(bot, avatar_source)
else:
return json_error(_("You may only upload one file at a time"))
json_result = dict(
full_name=bot.full_name,
avatar_url=avatar_url(bot),
default_sending_stream=get_stream_name(bot.default_sending_stream),
default_events_register_stream=get_stream_name(
bot.default_events_register_stream),
default_all_public_streams=bot.default_all_public_streams,
)
# Don't include the bot owner in case it is not set.
# Default bots have no owner.
if bot.bot_owner is not None:
json_result['bot_owner'] = bot.bot_owner.email
return json_success(json_result)
