def json_stream_exists(
request: HttpRequest,
user_profile: UserProfile,
stream_name: str = REQ("stream"),
autosubscribe: bool = REQ(json_validator=check_bool, default=False),
) -> HttpResponse:
check_stream_name(stream_name)
try:
(stream, sub) = access_stream_by_name(user_profile, stream_name)
except JsonableError as e:
raise ResourceNotFoundError(e.msg)
# access_stream functions return a subscription if and only if we
# are already subscribed.
result = {"subscribed": sub is not None}
# If we got here, we're either subscribed or the stream is public.
# So if we're not yet subscribed and autosubscribe is enabled, we
# should join.
if sub is None and autosubscribe:
bulk_add_subscriptions(user_profile.realm, [stream], [user_profile],
acting_user=user_profile)
result["subscribed"] = True
return json_success(request,
data=result) # results are ignored for HEAD requests
def test_subscriptions(self) -> None:
now = timezone_now()
user = self.example_user("hamlet")
realm = user.realm
stream = self.make_stream("test_stream")
acting_user = self.example_user("iago")
bulk_add_subscriptions(user.realm, [stream], [user], acting_user=acting_user)
subscription_creation_logs = RealmAuditLog.objects.filter(
event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
event_time__gte=now,
acting_user=acting_user,
modified_user=user,
modified_stream=stream,
)
modified_stream = subscription_creation_logs[0].modified_stream
assert modified_stream is not None
self.assertEqual(subscription_creation_logs.count(), 1)
self.assertEqual(modified_stream.id, stream.id)
self.assertEqual(subscription_creation_logs[0].modified_user, user)
bulk_remove_subscriptions(realm, [user], [stream], acting_user=acting_user)
subscription_deactivation_logs = RealmAuditLog.objects.filter(
event_type=RealmAuditLog.SUBSCRIPTION_DEACTIVATED,
event_time__gte=now,
acting_user=acting_user,
modified_user=user,
modified_stream=stream,
)
modified_stream = subscription_deactivation_logs[0].modified_stream
assert modified_stream is not None
self.assertEqual(subscription_deactivation_logs.count(), 1)
self.assertEqual(modified_stream.id, stream.id)
self.assertEqual(subscription_deactivation_logs[0].modified_user, user)
def handle(self, *args: Any, **options: str) -> None:
realm = self.get_realm(options)
assert realm is not None # Should be ensured by parser
stream_to_keep = get_stream(options["stream_to_keep"], realm)
stream_to_destroy = get_stream(options["stream_to_destroy"], realm)
recipient_to_destroy = stream_to_destroy.recipient
recipient_to_keep = stream_to_keep.recipient
# The high-level approach here is to move all the messages to
# the surviving stream, deactivate all the subscriptions on
# the stream to be removed and deactivate the stream, and add
# new subscriptions to the stream to keep for any users who
# were only on the now-deactivated stream.
# Move the messages, and delete the old copies from caches.
message_ids_to_clear = list(
Message.objects.filter(recipient=recipient_to_destroy).values_list("id", flat=True)
)
count = Message.objects.filter(recipient=recipient_to_destroy).update(
recipient=recipient_to_keep
)
print(f"Moved {count} messages")
bulk_delete_cache_keys(message_ids_to_clear)
# Move the Subscription objects. This algorithm doesn't
# preserve any stream settings/colors/etc. from the stream
# being destroyed, but it's convenient.
existing_subs = Subscription.objects.filter(recipient=recipient_to_keep)
users_already_subscribed = {sub.user_profile_id: sub.active for sub in existing_subs}
subs_to_deactivate = Subscription.objects.filter(
recipient=recipient_to_destroy, active=True
)
users_to_activate = [
sub.user_profile
for sub in subs_to_deactivate
if not users_already_subscribed.get(sub.user_profile_id, False)
]
if len(subs_to_deactivate) > 0:
print(f"Deactivating {len(subs_to_deactivate)} subscriptions")
bulk_remove_subscriptions(
realm,
[sub.user_profile for sub in subs_to_deactivate],
[stream_to_destroy],
acting_user=None,
)
do_deactivate_stream(stream_to_destroy, acting_user=None)
if len(users_to_activate) > 0:
print(f"Adding {len(users_to_activate)} subscriptions")
bulk_add_subscriptions(realm, [stream_to_keep], users_to_activate, acting_user=None)
def handle(self, *args: Any, **options: Any) -> None:
realm = self.get_realm(options)
assert realm is not None # Should be ensured by parser
user_profiles = self.get_users(options, realm)
stream_names = {
stream.strip()
for stream in options["streams"].split(",")
}
for stream_name in set(stream_names):
for user_profile in user_profiles:
stream = ensure_stream(realm, stream_name, acting_user=None)
_ignore, already_subscribed = bulk_add_subscriptions(
realm, [stream], [user_profile], acting_user=None)
was_there_already = user_profile.id in (
info.user.id for info in already_subscribed)
print("{} {} to {}".format(
"Already subscribed"
if was_there_already else "Subscribed",
user_profile.delivery_email,
stream_name,
))
def do_create_user(
email: str,
password: Optional[str],
realm: Realm,
full_name: str,
bot_type: Optional[int] = None,
role: Optional[int] = None,
bot_owner: Optional[UserProfile] = None,
tos_version: Optional[str] = None,
timezone: str = "",
avatar_source: str = UserProfile.AVATAR_FROM_GRAVATAR,
default_language: str = "en",
default_sending_stream: Optional[Stream] = None,
default_events_register_stream: Optional[Stream] = None,
default_all_public_streams: Optional[bool] = None,
prereg_user: Optional[PreregistrationUser] = None,
default_stream_groups: Sequence[DefaultStreamGroup] = [],
source_profile: Optional[UserProfile] = None,
realm_creation: bool = False,
*,
acting_user: Optional[UserProfile],
enable_marketing_emails: bool = True,
) -> UserProfile:
with transaction.atomic():
user_profile = create_user(
email=email,
password=password,
realm=realm,
full_name=full_name,
role=role,
bot_type=bot_type,
bot_owner=bot_owner,
tos_version=tos_version,
timezone=timezone,
avatar_source=avatar_source,
default_language=default_language,
default_sending_stream=default_sending_stream,
default_events_register_stream=default_events_register_stream,
default_all_public_streams=default_all_public_streams,
source_profile=source_profile,
enable_marketing_emails=enable_marketing_emails,
)
event_time = user_profile.date_joined
if not acting_user:
acting_user = user_profile
RealmAuditLog.objects.create(
realm=user_profile.realm,
acting_user=acting_user,
modified_user=user_profile,
event_type=RealmAuditLog.USER_CREATED,
event_time=event_time,
extra_data=orjson.dumps({
RealmAuditLog.ROLE_COUNT:
realm_user_count_by_role(user_profile.realm),
}).decode(),
)
if realm_creation:
# If this user just created a realm, make sure they are
# properly tagged as the creator of the realm.
realm_creation_audit_log = (RealmAuditLog.objects.filter(
event_type=RealmAuditLog.REALM_CREATED,
realm=realm).order_by("id").last())
assert realm_creation_audit_log is not None
realm_creation_audit_log.acting_user = user_profile
realm_creation_audit_log.save(update_fields=["acting_user"])
do_increment_logging_stat(
user_profile.realm,
COUNT_STATS["active_users_log:is_bot:day"],
user_profile.is_bot,
event_time,
)
if settings.BILLING_ENABLED:
update_license_ledger_if_needed(user_profile.realm, event_time)
system_user_group = get_system_user_group_for_user(user_profile)
UserGroupMembership.objects.create(user_profile=user_profile,
user_group=system_user_group)
if user_profile.role == UserProfile.ROLE_MEMBER and not user_profile.is_provisional_member:
full_members_system_group = UserGroup.objects.get(
name="@role:fullmembers",
realm=user_profile.realm,
is_system_group=True)
UserGroupMembership.objects.create(
user_profile=user_profile,
user_group=full_members_system_group)
# Note that for bots, the caller will send an additional event
# with bot-specific info like services.
notify_created_user(user_profile)
do_send_user_group_members_update_event("add_members", system_user_group,
[user_profile.id])
if user_profile.role == UserProfile.ROLE_MEMBER and not user_profile.is_provisional_member:
do_send_user_group_members_update_event("add_members",
full_members_system_group,
[user_profile.id])
if bot_type is None:
process_new_human_user(
user_profile,
prereg_user=prereg_user,
default_stream_groups=default_stream_groups,
realm_creation=realm_creation,
)
if realm_creation:
assert realm.signup_notifications_stream is not None
bulk_add_subscriptions(realm, [realm.signup_notifications_stream],
[user_profile],
acting_user=None)
from zerver.lib.onboarding import send_initial_realm_messages
send_initial_realm_messages(realm)
return user_profile
def process_new_human_user(
user_profile: UserProfile,
prereg_user: Optional[PreregistrationUser] = None,
default_stream_groups: Sequence[DefaultStreamGroup] = [],
realm_creation: bool = False,
) -> None:
realm = user_profile.realm
mit_beta_user = realm.is_zephyr_mirror_realm
if prereg_user is not None:
streams: List[Stream] = list(prereg_user.streams.all())
acting_user: Optional[UserProfile] = prereg_user.referred_by
# A PregistrationUser should not be used for another UserProfile
assert prereg_user.created_user is None, "PregistrationUser should not be reused"
else:
streams = []
acting_user = None
# If the user's invitation didn't explicitly list some streams, we
# add the default streams
if len(streams) == 0:
streams = get_default_subs(user_profile)
for default_stream_group in default_stream_groups:
default_stream_group_streams = default_stream_group.streams.all()
for stream in default_stream_group_streams:
if stream not in streams:
streams.append(stream)
bulk_add_subscriptions(
realm,
streams,
[user_profile],
from_user_creation=True,
acting_user=acting_user,
)
add_new_user_history(user_profile, streams)
# mit_beta_users don't have a referred_by field
if (not mit_beta_user and prereg_user is not None
and prereg_user.referred_by is not None
and prereg_user.referred_by.is_active):
# This is a cross-realm private message.
with override_language(prereg_user.referred_by.default_language):
internal_send_private_message(
get_system_bot(settings.NOTIFICATION_BOT,
prereg_user.referred_by.realm_id),
prereg_user.referred_by,
_("{user} accepted your invitation to join Zulip!").format(
user=f"{user_profile.full_name} <`{user_profile.email}`>"),
)
# Revoke all preregistration users except prereg_user, and link prereg_user to
# the created user
if prereg_user is None:
assert not realm_creation, "realm_creation should only happen with a PreregistrationUser"
if prereg_user is not None:
prereg_user.status = confirmation_settings.STATUS_ACTIVE
prereg_user.created_user = user_profile
prereg_user.save(update_fields=["status", "created_user"])
# In the special case of realm creation, there can be no additional PreregistrationUser
# for us to want to modify - because other realm_creation PreregistrationUsers should be
# left usable for creating different realms.
if not realm_creation:
# Mark any other PreregistrationUsers in the realm that are STATUS_ACTIVE as
# inactive so we can keep track of the PreregistrationUser we
# actually used for analytics.
if prereg_user is not None:
PreregistrationUser.objects.filter(
email__iexact=user_profile.delivery_email,
realm=user_profile.realm).exclude(id=prereg_user.id).update(
status=confirmation_settings.STATUS_REVOKED)
else:
PreregistrationUser.objects.filter(
email__iexact=user_profile.delivery_email,
realm=user_profile.realm).update(
status=confirmation_settings.STATUS_REVOKED)
if prereg_user is not None and prereg_user.referred_by is not None:
notify_invites_changed(user_profile.realm)
notify_new_user(user_profile)
# Clear any scheduled invitation emails to prevent them
# from being sent after the user is created.
clear_scheduled_invitation_emails(user_profile.delivery_email)
if realm.send_welcome_emails:
enqueue_welcome_emails(user_profile, realm_creation)
# We have an import loop here; it's intentional, because we want
# to keep all the onboarding code in zerver/lib/onboarding.py.
from zerver.lib.onboarding import send_initial_pms
send_initial_pms(user_profile)
def add_message_formatting_conversation(self) -> None:
realm = get_realm("zulip")
stream = ensure_stream(realm, "zulip features", acting_user=None)
UserProfile.objects.filter(email__contains="stage").delete()
starr = do_create_user("*****@*****.**",
"password",
realm,
"Ada Starr",
acting_user=None)
self.set_avatar(starr, "static/images/characters/starr.png")
fisher = do_create_user("*****@*****.**",
"password",
realm,
"Bel Fisher",
acting_user=None)
self.set_avatar(fisher, "static/images/characters/fisher.png")
twitter_bot = do_create_user(
"*****@*****.**",
"password",
realm,
"Twitter Bot",
bot_type=UserProfile.DEFAULT_BOT,
acting_user=None,
)
self.set_avatar(twitter_bot, "static/images/features/twitter.png")
bulk_add_subscriptions(realm, [stream],
list(UserProfile.objects.filter(realm=realm)),
acting_user=None)
staged_messages: List[Dict[str, Any]] = [
{
"sender":
starr,
"content":
"Hey @**Bel Fisher**, check out Zulip's Markdown formatting! "
"You can have:\n* bulleted lists\n * with sub-bullets too\n"
"* **bold**, *italic*, and ~~strikethrough~~ text\n"
"* LaTeX for mathematical formulas, both inline -- $$O(n^2)$$ -- and displayed:\n"
"```math\n\\int_a^b f(t)\\, dt=F(b)-F(a)\n```",
},
{
"sender":
fisher,
"content":
"My favorite is the syntax highlighting for code blocks\n"
"```python\ndef fib(n: int) -> int:\n # returns the n-th Fibonacci number\n"
" return fib(n-1) + fib(n-2)\n```",
},
{
"sender":
starr,
"content":
"I think you forgot your base case there, Bel :laughing:\n"
"```quote\n```python\ndef fib(n: int) -> int:\n # returns the n-th Fibonacci number\n"
" return fib(n-1) + fib(n-2)\n```\n```",
},
{
"sender":
fisher,
"content":
"I'm also a big fan of inline link, tweet, video, and image previews. "
"Check out this picture of Çet Whalin[](/static/images/features/whale.png)!",
},
{
"sender":
starr,
"content":
"I just set up a custom linkifier, "
"so `#1234` becomes [#1234](github.com/zulip/zulip/1234), "
"a link to the corresponding GitHub issue.",
},
{
"sender":
twitter_bot,
"content":
"https://twitter.com/gvanrossum/status/786661035637772288",
},
{
"sender":
fisher,
"content":
"Oops, the Twitter bot I set up shouldn't be posting here. Let me go fix that.",
},
]
messages = [
internal_prep_stream_message(
message["sender"],
stream,
"message formatting",
message["content"],
) for message in staged_messages
]
message_ids = do_send_messages(messages)
preview_message = Message.objects.get(
id__in=message_ids, content__icontains="image previews")
(emoji_code, reaction_type) = emoji_name_to_emoji_code(realm, "whale")
do_add_reaction(starr, preview_message, "whale", emoji_code,
reaction_type)
twitter_message = Message.objects.get(id__in=message_ids,
content__icontains="gvanrossum")
# Setting up a twitter integration in dev is a decent amount of work. If you need
# to update this tweet, either copy the format below, or send the link to the tweet
# to chat.zulip.org and ask an admin of that server to get you the rendered_content.
twitter_message.rendered_content = (
"<p><a>https://twitter.com/gvanrossum/status/786661035637772288</a></p>\n"
'<div class="inline-preview-twitter"><div class="twitter-tweet">'
'<a><img class="twitter-avatar" '
'src="https://pbs.twimg.com/profile_images/424495004/GuidoAvatar_bigger.jpg"></a>'
"<p>Great blog post about Zulip's use of mypy: "
"<a>http://blog.zulip.org/2016/10/13/static-types-in-python-oh-mypy/</a></p>"
"<span>- Guido van Rossum (@gvanrossum)</span></div></div>")
twitter_message.save(update_fields=["rendered_content"])
# Put a short pause between the whale reaction and this, so that the
# thumbs_up shows up second
(emoji_code,
reaction_type) = emoji_name_to_emoji_code(realm, "thumbs_up")
do_add_reaction(starr, preview_message, "thumbs_up", emoji_code,
reaction_type)
def add_subscriptions_backend(
request: HttpRequest,
user_profile: UserProfile,
streams_raw: Sequence[Mapping[str, str]] = REQ(
"subscriptions", json_validator=add_subscriptions_schema),
invite_only: bool = REQ(json_validator=check_bool, default=False),
is_web_public: bool = REQ(json_validator=check_bool, default=False),
stream_post_policy: int = REQ(
json_validator=check_int_in(Stream.STREAM_POST_POLICY_TYPES),
default=Stream.STREAM_POST_POLICY_EVERYONE,
),
history_public_to_subscribers: Optional[bool] = REQ(
json_validator=check_bool, default=None),
message_retention_days: Union[str, int] = REQ(
json_validator=check_string_or_int, default=RETENTION_DEFAULT),
announce: bool = REQ(json_validator=check_bool, default=False),
principals: Union[Sequence[str], Sequence[int]] = REQ(
json_validator=check_principals,
default=EMPTY_PRINCIPALS,
),
authorization_errors_fatal: bool = REQ(json_validator=check_bool,
default=True),
) -> HttpResponse:
realm = user_profile.realm
stream_dicts = []
color_map = {}
for stream_dict in streams_raw:
# 'color' field is optional
# check for its presence in the streams_raw first
if "color" in stream_dict:
color_map[stream_dict["name"]] = stream_dict["color"]
stream_dict_copy: StreamDict = {}
stream_dict_copy["name"] = stream_dict["name"].strip()
# We don't allow newline characters in stream descriptions.
if "description" in stream_dict:
stream_dict_copy["description"] = stream_dict[
"description"].replace("\n", " ")
stream_dict_copy["invite_only"] = invite_only
stream_dict_copy["is_web_public"] = is_web_public
stream_dict_copy["stream_post_policy"] = stream_post_policy
stream_dict_copy[
"history_public_to_subscribers"] = history_public_to_subscribers
stream_dict_copy[
"message_retention_days"] = parse_message_retention_days(
message_retention_days,
Stream.MESSAGE_RETENTION_SPECIAL_VALUES_MAP)
stream_dicts.append(stream_dict_copy)
# Validation of the streams arguments, including enforcement of
# can_create_streams policy and check_stream_name policy is inside
# list_to_streams.
existing_streams, created_streams = list_to_streams(stream_dicts,
user_profile,
autocreate=True)
authorized_streams, unauthorized_streams = filter_stream_authorization(
user_profile, existing_streams)
if len(unauthorized_streams) > 0 and authorization_errors_fatal:
raise JsonableError(
_("Unable to access stream ({stream_name}).").format(
stream_name=unauthorized_streams[0].name, ))
# Newly created streams are also authorized for the creator
streams = authorized_streams + created_streams
if len(principals) > 0:
if realm.is_zephyr_mirror_realm and not all(stream.invite_only
for stream in streams):
raise JsonableError(
_("You can only invite other Zephyr mirroring users to private streams."
))
if not user_profile.can_subscribe_other_users():
# Guest users case will not be handled here as it will
# be handled by the decorator above.
raise JsonableError(_("Insufficient permission"))
subscribers = {
principal_to_user_profile(user_profile, principal)
for principal in principals
}
else:
subscribers = {user_profile}
(subscribed,
already_subscribed) = bulk_add_subscriptions(realm,
streams,
subscribers,
acting_user=user_profile,
color_map=color_map)
# We can assume unique emails here for now, but we should eventually
# convert this function to be more id-centric.
email_to_user_profile: Dict[str, UserProfile] = {}
result: Dict[str, Any] = dict(subscribed=defaultdict(list),
already_subscribed=defaultdict(list))
for sub_info in subscribed:
subscriber = sub_info.user
stream = sub_info.stream
result["subscribed"][subscriber.email].append(stream.name)
email_to_user_profile[subscriber.email] = subscriber
for sub_info in already_subscribed:
subscriber = sub_info.user
stream = sub_info.stream
result["already_subscribed"][subscriber.email].append(stream.name)
result["subscribed"] = dict(result["subscribed"])
result["already_subscribed"] = dict(result["already_subscribed"])
send_messages_for_new_subscribers(
user_profile=user_profile,
subscribers=subscribers,
new_subscriptions=result["subscribed"],
email_to_user_profile=email_to_user_profile,
created_streams=created_streams,
announce=announce,
)
result["subscribed"] = dict(result["subscribed"])
result["already_subscribed"] = dict(result["already_subscribed"])
if not authorization_errors_fatal:
result["unauthorized"] = [s.name for s in unauthorized_streams]
return json_success(request, data=result)
def handle(self, *args: Any, **options: Any) -> None:
# Suppress spammy output from the push notifications logger
push_notifications_logger.disabled = True
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write("Error! More than 100% of messages allocated.\n")
return
# Get consistent data for backend tests.
if options["test_suite"]:
random.seed(0)
with connection.cursor() as cursor:
# Sometimes bugs relating to confusing recipient.id for recipient.type_id
# or <object>.id for <object>.recipient_id remain undiscovered by the test suite
# due to these numbers happening to coincide in such a way that it makes tests
# accidentally pass. By bumping the Recipient.id sequence by a large enough number,
# we can have those ids in a completely different range of values than object ids,
# eliminating the possibility of such coincidences.
cursor.execute("SELECT setval('zerver_recipient_id_seq', 100)")
if options["max_topics"] is None:
# If max_topics is not set, we use a default that's big
# enough "more topics" should appear, and scales slowly
# with the number of messages.
options["max_topics"] = 8 + options["num_messages"] // 1000
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our three default realms
# Could in theory be done via zerver.actions.create_realm.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
create_internal_realm()
zulip_realm = do_create_realm(
string_id="zulip",
name="Zulip Dev",
emails_restricted_to_domains=False,
email_address_visibility=Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS,
description="The Zulip development environment default organization."
" It's great for testing!",
invite_required=False,
plan_type=Realm.PLAN_TYPE_SELF_HOSTED,
org_type=Realm.ORG_TYPES["business"]["id"],
enable_spectator_access=True,
)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
assert zulip_realm.notifications_stream is not None
zulip_realm.notifications_stream.name = "Verona"
zulip_realm.notifications_stream.description = "A city in Italy"
zulip_realm.notifications_stream.save(update_fields=["name", "description"])
realm_user_default = RealmUserDefault.objects.get(realm=zulip_realm)
realm_user_default.enter_sends = True
realm_user_default.save()
if options["test_suite"]:
mit_realm = do_create_realm(
string_id="zephyr",
name="MIT",
emails_restricted_to_domains=True,
invite_required=False,
plan_type=Realm.PLAN_TYPE_SELF_HOSTED,
org_type=Realm.ORG_TYPES["business"]["id"],
)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = do_create_realm(
string_id="lear",
name="Lear & Co.",
emails_restricted_to_domains=False,
invite_required=False,
plan_type=Realm.PLAN_TYPE_SELF_HOSTED,
org_type=Realm.ORG_TYPES["business"]["id"],
)
# Default to allowing all members to send mentions in
# large streams for the test suite to keep
# mention-related tests simple.
zulip_realm.wildcard_mention_policy = Realm.WILDCARD_MENTION_POLICY_MEMBERS
zulip_realm.save(update_fields=["wildcard_mention_policy"])
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia, Lear's daughter", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
("Polonius", "*****@*****.**"),
("Desdemona", "*****@*****.**"),
("शिव", "*****@*****.**"),
]
# For testing really large batches:
# Create extra users with semi realistic names to make search
# functions somewhat realistic. We'll still create 1000 users
# like Extra222 User for some predictability.
num_names = options["extra_users"]
num_boring_names = 300
for i in range(min(num_names, num_boring_names)):
full_name = f"Extra{i:03} User"
names.append((full_name, f"extrauser{i}@zulip.com"))
if num_names > num_boring_names:
fnames = [
"Amber",
"Arpita",
"Bob",
"Cindy",
"Daniela",
"Dan",
"Dinesh",
"Faye",
"François",
"George",
"Hank",
"Irene",
"James",
"Janice",
"Jenny",
"Jill",
"John",
"Kate",
"Katelyn",
"Kobe",
"Lexi",
"Manish",
"Mark",
"Matt",
"Mayna",
"Michael",
"Pete",
"Peter",
"Phil",
"Phillipa",
"Preston",
"Sally",
"Scott",
"Sandra",
"Steve",
"Stephanie",
"Vera",
]
mnames = ["de", "van", "von", "Shaw", "T."]
lnames = [
"Adams",
"Agarwal",
"Beal",
"Benson",
"Bonita",
"Davis",
"George",
"Harden",
"James",
"Jones",
"Johnson",
"Jordan",
"Lee",
"Leonard",
"Singh",
"Smith",
"Patel",
"Towns",
"Wall",
]
non_ascii_names = [
"Günter",
"أحمد",
"Magnús",
"आशी",
"イツキ",
"语嫣",
"அருண்",
"Александр",
"José",
]
# to imitate emoji insertions in usernames
raw_emojis = ["😎", "😂", "🐱👤"]
for i in range(num_boring_names, num_names):
fname = random.choice(fnames) + str(i)
full_name = fname
if random.random() < 0.7:
if random.random() < 0.3:
full_name += " " + random.choice(non_ascii_names)
else:
full_name += " " + random.choice(mnames)
if random.random() < 0.1:
full_name += f" {random.choice(raw_emojis)} "
else:
full_name += " " + random.choice(lnames)
email = fname.lower() + "@zulip.com"
names.append((full_name, email))
create_users(zulip_realm, names, tos_version=settings.TERMS_OF_SERVICE_VERSION)
# Add time zones to some users. Ideally, this would be
# done in the initial create_users calls, but the
# tuple-based interface for that function doesn't support
# doing so.
def assign_time_zone_by_delivery_email(delivery_email: str, new_time_zone: str) -> None:
u = get_user_by_delivery_email(delivery_email, zulip_realm)
u.timezone = new_time_zone
u.save(update_fields=["timezone"])
# Note: Hamlet keeps default time zone of "".
assign_time_zone_by_delivery_email("*****@*****.**", "US/Pacific")
assign_time_zone_by_delivery_email("*****@*****.**", "US/Pacific")
assign_time_zone_by_delivery_email("*****@*****.**", "US/Eastern")
assign_time_zone_by_delivery_email("*****@*****.**", "US/Eastern")
assign_time_zone_by_delivery_email("*****@*****.**", "Canada/Newfoundland")
assign_time_zone_by_delivery_email("*****@*****.**", "Asia/Shanghai") # China
assign_time_zone_by_delivery_email("*****@*****.**", "Asia/Kolkata") # India
assign_time_zone_by_delivery_email("*****@*****.**", "UTC")
users = UserProfile.objects.filter(realm=zulip_realm)
# All users in development environment are full members initially because
# waiting period threshold is 0. Groups of Iago, Dedemona, Shiva and
# Polonius will be updated according to their role in do_change_user_role.
full_members_user_group = UserGroup.objects.get(
realm=zulip_realm, name="@role:fullmembers", is_system_group=True
)
members_user_group = UserGroup.objects.get(
realm=zulip_realm, name="@role:members", is_system_group=True
)
user_group_memberships = []
for user_profile in list(users):
for group in [full_members_user_group, members_user_group]:
user_group_membership = UserGroupMembership(
user_group=group, user_profile=user_profile
)
user_group_memberships.append(user_group_membership)
UserGroupMembership.objects.bulk_create(user_group_memberships)
iago = get_user_by_delivery_email("*****@*****.**", zulip_realm)
do_change_user_role(iago, UserProfile.ROLE_REALM_ADMINISTRATOR, acting_user=None)
iago.is_staff = True
iago.save(update_fields=["is_staff"])
# We need to create at least two test draft for Iago for the sake
# of the cURL tests. Two since one will be deleted.
Draft.objects.create(
user_profile=iago,
recipient=None,
topic="Release Notes",
content="Release 4.0 will contain ...",
last_edit_time=timezone_now(),
)
Draft.objects.create(
user_profile=iago,
recipient=None,
topic="Release Notes",
content="Release 4.0 will contain many new features such as ... ",
last_edit_time=timezone_now(),
)
desdemona = get_user_by_delivery_email("*****@*****.**", zulip_realm)
do_change_user_role(desdemona, UserProfile.ROLE_REALM_OWNER, acting_user=None)
shiva = get_user_by_delivery_email("*****@*****.**", zulip_realm)
do_change_user_role(shiva, UserProfile.ROLE_MODERATOR, acting_user=None)
polonius = get_user_by_delivery_email("*****@*****.**", zulip_realm)
do_change_user_role(polonius, UserProfile.ROLE_GUEST, acting_user=None)
# These bots are directly referenced from code and thus
# are needed for the test suite.
zulip_realm_bots = [
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append((f"Extra Bot {i}", f"extrabot{i}@zulip.com"))
create_users(
zulip_realm, zulip_realm_bots, bot_type=UserProfile.DEFAULT_BOT, bot_owner=desdemona
)
zoe = get_user_by_delivery_email("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(
zulip_realm,
zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
bot_owner=zoe,
)
aaron = get_user_by_delivery_email("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [
("Outgoing Webhook", "*****@*****.**"),
]
create_users(
zulip_realm,
zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
bot_owner=aaron,
)
outgoing_webhook = get_user("*****@*****.**", zulip_realm)
add_service(
"outgoing-webhook",
user_profile=outgoing_webhook,
interface=Service.GENERIC,
base_url="http://127.0.0.1:5002",
token=generate_api_key(),
)
# Add the realm internal bots to each realm.
create_if_missing_realm_internal_bots()
# Create public streams.
signups_stream = Realm.INITIAL_PRIVATE_STREAM_NAME
stream_list = [
"Verona",
"Denmark",
"Scotland",
"Venice",
"Rome",
signups_stream,
]
stream_dict: Dict[str, Dict[str, Any]] = {
"Denmark": {"description": "A Scandinavian country"},
"Scotland": {"description": "Located in the United Kingdom"},
"Venice": {"description": "A northeastern Italian city"},
"Rome": {"description": "Yet another Italian city", "is_web_public": True},
}
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams: List[int] = [
Stream.objects.get(name=name, realm=zulip_realm).id for name in stream_list
]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users). For the test suite, we have a fixed list of
# subscriptions to make sure test data is consistent
# across platforms.
subscriptions_list: List[Tuple[UserProfile, Recipient]] = []
profiles: Sequence[UserProfile] = list(
UserProfile.objects.select_related().filter(is_bot=False).order_by("email")
)
if options["test_suite"]:
subscriptions_map = {
"*****@*****.**": ["Verona"],
"*****@*****.**": ["Verona"],
"*****@*****.**": ["Verona", "Denmark", signups_stream],
"*****@*****.**": [
"Verona",
"Denmark",
"Scotland",
signups_stream,
],
"*****@*****.**": ["Verona", "Denmark", "Scotland"],
"*****@*****.**": ["Verona", "Denmark", "Scotland", "Venice"],
"*****@*****.**": ["Verona", "Denmark", "Scotland", "Venice", "Rome"],
"*****@*****.**": ["Verona"],
"*****@*****.**": [
"Verona",
"Denmark",
"Venice",
signups_stream,
],
"*****@*****.**": ["Verona", "Denmark", "Scotland"],
}
for profile in profiles:
email = profile.delivery_email
if email not in subscriptions_map:
raise Exception(f"Subscriptions not listed for user {email}")
for stream_name in subscriptions_map[email]:
stream = Stream.objects.get(name=stream_name, realm=zulip_realm)
r = Recipient.objects.get(type=Recipient.STREAM, type_id=stream.id)
subscriptions_list.append((profile, r))
else:
num_streams = len(recipient_streams)
num_users = len(profiles)
for i, profile in enumerate(profiles):
# Subscribe to some streams.
fraction = float(i) / num_users
num_recips = int(num_streams * fraction) + 1
for type_id in recipient_streams[:num_recips]:
r = Recipient.objects.get(type=Recipient.STREAM, type_id=type_id)
subscriptions_list.append((profile, r))
subscriptions_to_add: List[Subscription] = []
event_time = timezone_now()
all_subscription_logs: (List[RealmAuditLog]) = []
i = 0
for profile, recipient in subscriptions_list:
i += 1
color = STREAM_ASSIGNMENT_COLORS[i % len(STREAM_ASSIGNMENT_COLORS)]
s = Subscription(
recipient=recipient,
user_profile=profile,
is_user_active=profile.is_active,
color=color,
)
subscriptions_to_add.append(s)
log = RealmAuditLog(
realm=profile.realm,
modified_user=profile,
modified_stream_id=recipient.type_id,
event_last_message_id=0,
event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
event_time=event_time,
)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(
zulip_realm, "Phone number", CustomProfileField.SHORT_TEXT, hint=""
)
biography = try_add_realm_custom_profile_field(
zulip_realm,
"Biography",
CustomProfileField.LONG_TEXT,
hint="What are you known for?",
)
favorite_food = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite food",
CustomProfileField.SHORT_TEXT,
hint="Or drink, if you'd prefer",
)
field_data: ProfileFieldData = {
"vim": {"text": "Vim", "order": "1"},
"emacs": {"text": "Emacs", "order": "2"},
}
favorite_editor = try_add_realm_custom_profile_field(
zulip_realm, "Favorite editor", CustomProfileField.SELECT, field_data=field_data
)
birthday = try_add_realm_custom_profile_field(
zulip_realm, "Birthday", CustomProfileField.DATE
)
favorite_website = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite website",
CustomProfileField.URL,
hint="Or your personal blog's URL",
)
mentor = try_add_realm_custom_profile_field(
zulip_realm, "Mentor", CustomProfileField.USER
)
github_profile = try_add_realm_default_custom_profile_field(zulip_realm, "github")
# Fill in values for Iago and Hamlet
hamlet = get_user_by_delivery_email("*****@*****.**", zulip_realm)
do_update_user_custom_profile_data_if_changed(
iago,
[
{"id": phone_number.id, "value": "+1-234-567-8901"},
{"id": biography.id, "value": "Betrayer of Othello."},
{"id": favorite_food.id, "value": "Apples"},
{"id": favorite_editor.id, "value": "emacs"},
{"id": birthday.id, "value": "2000-01-01"},
{"id": favorite_website.id, "value": "https://zulip.readthedocs.io/en/latest/"},
{"id": mentor.id, "value": [hamlet.id]},
{"id": github_profile.id, "value": "zulip"},
],
)
do_update_user_custom_profile_data_if_changed(
hamlet,
[
{"id": phone_number.id, "value": "+0-11-23-456-7890"},
{
"id": biography.id,
"value": "I am:\n* The prince of Denmark\n* Nephew to the usurping Claudius",
},
{"id": favorite_food.id, "value": "Dark chocolate"},
{"id": favorite_editor.id, "value": "vim"},
{"id": birthday.id, "value": "1900-01-01"},
{"id": favorite_website.id, "value": "https://blog.zulig.org"},
{"id": mentor.id, "value": [iago.id]},
{"id": github_profile.id, "value": "zulipbot"},
],
)
else:
zulip_realm = get_realm("zulip")
recipient_streams = [
klass.type_id for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
# Extract a list of all users
user_profiles: List[UserProfile] = list(UserProfile.objects.filter(is_bot=False))
# Create a test realm emoji.
IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
with open(IMAGE_FILE_PATH, "rb") as fp:
check_add_realm_emoji(zulip_realm, "green_tick", iago, File(fp))
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status: int = UserPresence.ACTIVE
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= "H":
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(
user_profile=user,
realm_id=user.realm_id,
client=client,
timestamp=date,
status=status,
)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [
random.sample(user_profiles_ids, 2) for i in range(options["num_personals"])
]
create_alert_words(zulip_realm.id)
# Generate a new set of test data.
create_test_data()
if options["delete"]:
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
event_time = timezone_now()
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)", "*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(
mit_realm, testsuite_mit_users, tos_version=settings.TERMS_OF_SERVICE_VERSION
)
mit_user = get_user_by_delivery_email("*****@*****.**", mit_realm)
mit_signup_stream = Stream.objects.get(
name=Realm.INITIAL_PRIVATE_STREAM_NAME, realm=mit_realm
)
bulk_add_subscriptions(mit_realm, [mit_signup_stream], [mit_user], acting_user=None)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia, Lear's daughter", "*****@*****.**"),
]
create_users(
lear_realm, testsuite_lear_users, tos_version=settings.TERMS_OF_SERVICE_VERSION
)
lear_user = get_user_by_delivery_email("*****@*****.**", lear_realm)
lear_signup_stream = Stream.objects.get(
name=Realm.INITIAL_PRIVATE_STREAM_NAME, realm=lear_realm
)
bulk_add_subscriptions(
lear_realm, [lear_signup_stream], [lear_user], acting_user=None
)
if not options["test_suite"]:
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
# to imitate emoji insertions in stream names
raw_emojis = ["😎", "😂", "🐱👤"]
zulip_stream_dict: Dict[str, Dict[str, Any]] = {
"devel": {"description": "For developing"},
# ビデオゲーム - VideoGames (japanese)
"ビデオゲーム": {"description": f"Share your favorite video games! {raw_emojis[2]}"},
"announce": {
"description": "For announcements",
"stream_post_policy": Stream.STREAM_POST_POLICY_ADMINS,
},
"design": {"description": "For design"},
"support": {"description": "For support"},
"social": {"description": "For socializing"},
"test": {"description": "For testing `code`"},
"errors": {"description": "For errors"},
# 조리법 - Recipes (Korean) , Пельмени - Dumplings (Russian)
"조리법 "
+ raw_emojis[0]: {"description": "Everything cooking, from pasta to Пельмени"},
}
extra_stream_names = [
"802.11a",
"Ad Hoc Network",
"Augmented Reality",
"Cycling",
"DPI",
"FAQ",
"FiFo",
"commits",
"Control panel",
"desktop",
"компьютеры",
"Data security",
"desktop",
"काम",
"discussions",
"Cloud storage",
"GCI",
"Vaporware",
"Recent Trends",
"issues",
"live",
"Health",
"mobile",
"空間",
"provision",
"hidrógeno",
"HR",
"アニメ",
]
# Add stream names and stream descriptions
for i in range(options["extra_streams"]):
extra_stream_name = random.choice(extra_stream_names) + " " + str(i)
# to imitate emoji insertions in stream names
if random.random() <= 0.15:
extra_stream_name += random.choice(raw_emojis)
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream("announce", zulip_realm)
zulip_realm.save(update_fields=["notifications_stream"])
# Add a few default streams
for default_stream_name in ["design", "devel", "social", "support"]:
DefaultStream.objects.create(
realm=zulip_realm, stream=get_stream(default_stream_name, zulip_realm)
)
# Now subscribe everyone to these streams
subscribe_users_to_streams(zulip_realm, zulip_stream_dict)
create_user_groups()
if not options["test_suite"]:
# We populate the analytics database here for
# development purpose only
call_command("populate_analytics_db")
threads = options["threads"]
jobs: List[Tuple[int, List[List[int]], Dict[str, Any], int]] = []
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, random.randint(0, 10**10)))
for job in jobs:
generate_and_send_messages(job)
if options["delete"]:
if not options["test_suite"]:
# These bots are not needed by the test suite
# Also, we don't want interacting with each other
# in dev setup.
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(
zulip_realm,
internal_zulip_users_nosubs,
bot_type=UserProfile.DEFAULT_BOT,
bot_owner=desdemona,
)
mark_all_messages_as_read()
self.stdout.write("Successfully populated test database.\n")
push_notifications_logger.disabled = False