def check_stream_name_available(realm: Realm, name: str) -> None:
check_stream_name(name)
try:
get_stream(name, realm)
raise JsonableError(_("Stream name '%s' is already taken.") % (name,))
except Stream.DoesNotExist:
pass
def check_stream_name_available(realm, name):
# type: (Realm, Text) -> None
check_stream_name(name)
try:
get_stream(name, realm)
raise JsonableError(_("Stream name '%s' is already taken") % (name,))
except Stream.DoesNotExist:
pass
def test_signup_notifications_stream(self) -> None:
email = self.example_email("hamlet")
realm = get_realm('zulip')
realm.signup_notifications_stream = get_stream('Denmark', realm)
realm.save()
self.login(email)
result = self._get_home_page()
page_params = self._get_page_params(result)
self.assertEqual(page_params['realm_signup_notifications_stream_id'], get_stream('Denmark', realm).id)
def setUpBeforeMigration(self, apps: StateApps) -> None:
Reaction = apps.get_model('zerver', 'Reaction')
RealmEmoji = apps.get_model('zerver', 'RealmEmoji')
Message = apps.get_model('zerver', 'Message')
Recipient = apps.get_model('zerver', 'Recipient')
sender = self.example_user('iago')
realm = sender.realm
sending_client = make_client(name="test suite")
stream_name = 'Denmark'
stream = get_stream(stream_name, realm)
subject = 'foo'
def send_fake_message(message_content: str, stream: ModelBase) -> ModelBase:
recipient = Recipient.objects.get(type_id=stream.id, type=2)
return Message.objects.create(sender = sender,
recipient = recipient,
subject = subject,
content = message_content,
pub_date = timezone_now(),
sending_client = sending_client)
message = send_fake_message('Test 1', stream)
# Create reactions for all the realm emoji's on the message we faked.
for realm_emoji in RealmEmoji.objects.all():
reaction = Reaction(user_profile=sender, message=message,
emoji_name=realm_emoji.name, emoji_code=realm_emoji.name,
reaction_type='realm_emoji')
reaction.save()
realm_emoji_reactions_count = Reaction.objects.filter(reaction_type='realm_emoji').count()
self.assertEqual(realm_emoji_reactions_count, 1)
def test_receive_stream_email_messages_success(self) -> None:
# build dummy messages for stream
# test valid incoming stream message is processed properly
user_profile = self.example_user('hamlet')
self.login(user_profile.email)
self.subscribe(user_profile, "Denmark")
stream = get_stream("Denmark", user_profile.realm)
stream_to_address = encode_email_address(stream)
incoming_valid_message = MIMEText('TestStreamEmailMessages Body') # type: Any # https://github.com/python/typeshed/issues/275
incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
incoming_valid_message['From'] = self.example_email('hamlet')
incoming_valid_message['To'] = stream_to_address
incoming_valid_message['Reply-to'] = self.example_email('othello')
process_message(incoming_valid_message)
# Hamlet is subscribed to this stream so should see the email message from Othello.
message = most_recent_message(user_profile)
self.assertEqual(message.content, "TestStreamEmailMessages Body")
self.assertEqual(get_display_recipient(message.recipient), stream.name)
self.assertEqual(message.topic_name(), incoming_valid_message['Subject'])
def test_user_ids_muting_topic(self) -> None:
hamlet = self.example_user('hamlet')
cordelia = self.example_user('cordelia')
realm = hamlet.realm
stream = get_stream(u'Verona', realm)
recipient = get_stream_recipient(stream.id)
topic_name = 'teST topic'
stream_topic_target = StreamTopicTarget(
stream_id=stream.id,
topic_name=topic_name,
)
user_ids = stream_topic_target.user_ids_muting_topic()
self.assertEqual(user_ids, set())
def mute_user(user: UserProfile) -> None:
add_topic_mute(
user_profile=user,
stream_id=stream.id,
recipient_id=recipient.id,
topic_name='test TOPIC',
)
mute_user(hamlet)
user_ids = stream_topic_target.user_ids_muting_topic()
self.assertEqual(user_ids, {hamlet.id})
mute_user(cordelia)
user_ids = stream_topic_target.user_ids_muting_topic()
self.assertEqual(user_ids, {hamlet.id, cordelia.id})
def test_encode_email_addr(self):
# type: () -> None
stream = get_stream("Denmark", get_realm("zulip"))
with self.settings(EMAIL_GATEWAY_PATTERN=''):
test_address = encode_email_address(stream)
self.assertEqual(test_address, '')
def test_reply_is_extracted_from_plain(self) -> None:
# build dummy messages for stream
# test valid incoming stream message is processed properly
email = self.example_email('hamlet')
self.login(email)
user_profile = self.example_user('hamlet')
self.subscribe(user_profile, "Denmark")
stream = get_stream("Denmark", user_profile.realm)
stream_to_address = encode_email_address(stream)
text = """Reply
-----Original Message-----
Quote"""
incoming_valid_message = MIMEText(text) # type: Any # https://github.com/python/typeshed/issues/275
incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
incoming_valid_message['From'] = self.example_email('hamlet')
incoming_valid_message['To'] = stream_to_address
incoming_valid_message['Reply-to'] = self.example_email('othello')
process_message(incoming_valid_message)
# Hamlet is subscribed to this stream so should see the email message from Othello.
message = most_recent_message(user_profile)
self.assertEqual(message.content, "Reply")
def handle(self, **options):
# type: (*Any, **Any) -> None
if options["domain"] is None or options["stream"] is None or \
(options["users"] is None and options["all_users"] is None):
self.print_help("./manage.py", "remove_users_from_stream")
exit(1)
realm = get_realm(options["domain"])
stream_name = options["stream"].strip()
stream = get_stream(stream_name, realm)
if options["all_users"]:
user_profiles = UserProfile.objects.filter(realm=realm)
else:
emails = set([email.strip() for email in options["users"].split(",")])
user_profiles = []
for email in emails:
user_profiles.append(get_user_profile_by_email(email))
result = bulk_remove_subscriptions(user_profiles, [stream])
not_subscribed = result[1]
not_subscribed_users = {tup[0] for tup in not_subscribed}
for user_profile in user_profiles:
if user_profile in not_subscribed_users:
print("%s was not subscribed" % (user_profile.email,))
else:
print("Removed %s from %s" % (user_profile.email, stream_name))
def test_muted_topic_remove_invalid(self) -> None:
user = self.example_user('hamlet')
email = user.email
realm = user.realm
self.login(email)
stream = get_stream('Verona', realm)
url = '/api/v1/users/me/subscriptions/muted_topics'
data = {'stream': 'BOGUS', 'topic': 'Verona3', 'op': 'remove'} # type: Dict[str, Any]
result = self.api_patch(email, url, data)
self.assert_json_error(result, "Topic is not muted")
data = {'stream': stream.name, 'topic': 'BOGUS', 'op': 'remove'}
result = self.api_patch(email, url, data)
self.assert_json_error(result, "Topic is not muted")
data = {'stream_id': 999999999, 'topic': 'BOGUS', 'op': 'remove'}
result = self.api_patch(email, url, data)
self.assert_json_error(result, "Topic is not muted")
data = {'topic': 'Verona3', 'op': 'remove'}
result = self.api_patch(email, url, data)
self.assert_json_error(result, "Please supply 'stream'.")
data = {'stream': stream.name, 'stream_id': stream.id, 'topic': 'Verona3', 'op': 'remove'}
result = self.api_patch(email, url, data)
self.assert_json_error(result, "Please choose one: 'stream' or 'stream_id'.")
def test_add_muted_topic(self) -> None:
user = self.example_user('hamlet')
self.login(user.email)
stream = get_stream('Verona', user.realm)
url = '/api/v1/users/me/subscriptions/muted_topics'
payloads = [
{'stream': stream.name, 'topic': 'Verona3', 'op': 'add'},
{'stream_id': stream.id, 'topic': 'Verona3', 'op': 'add'},
]
for data in payloads:
result = self.api_patch(user.email, url, data)
self.assert_json_success(result)
self.assertIn([stream.name, 'Verona3'], get_topic_mutes(user))
self.assertTrue(topic_is_muted(user, stream.id, 'Verona3'))
self.assertTrue(topic_is_muted(user, stream.id, 'verona3'))
remove_topic_mute(
user_profile=user,
stream_id=stream.id,
topic_name='Verona3',
)
def test_muted_topic_add_invalid(self) -> None:
user = self.example_user('hamlet')
email = user.email
realm = user.realm
self.login(email)
stream = get_stream('Verona', realm)
recipient = get_stream_recipient(stream.id)
add_topic_mute(
user_profile=user,
stream_id=stream.id,
recipient_id=recipient.id,
topic_name=u'Verona3',
)
url = '/api/v1/users/me/subscriptions/muted_topics'
data = {'stream': stream.name, 'topic': 'Verona3', 'op': 'add'} # type: Dict[str, Any]
result = self.api_patch(email, url, data)
self.assert_json_error(result, "Topic already muted")
data = {'stream_id': 999999999, 'topic': 'Verona3', 'op': 'add'}
result = self.api_patch(email, url, data)
self.assert_json_error(result, "Invalid stream id")
data = {'topic': 'Verona3', 'op': 'add'}
result = self.api_patch(email, url, data)
self.assert_json_error(result, "Please supply 'stream'.")
data = {'stream': stream.name, 'stream_id': stream.id, 'topic': 'Verona3', 'op': 'add'}
result = self.api_patch(email, url, data)
self.assert_json_error(result, "Please choose one: 'stream' or 'stream_id'.")
def test_receive_stream_email_messages_no_textual_body(self) -> None:
user_profile = self.example_user('hamlet')
self.login(user_profile.email)
self.subscribe(user_profile, "Denmark")
stream = get_stream("Denmark", user_profile.realm)
stream_to_address = encode_email_address(stream)
headers = {}
headers['Reply-To'] = self.example_email('othello')
# No textual body
incoming_valid_message = MIMEMultipart()
with open(os.path.join(settings.DEPLOY_ROOT, "static/images/default-avatar.png"), 'rb') as f:
incoming_valid_message.attach(MIMEImage(f.read()))
incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
incoming_valid_message['From'] = self.example_email('hamlet')
incoming_valid_message['To'] = stream_to_address
incoming_valid_message['Reply-to'] = self.example_email('othello')
exception_message = ""
debug_info = {} # type: Dict[str, Any]
# process_message eats the exception & logs an error which can't be parsed here
# so calling process_stream_message directly
try:
with mock.patch('logging.warning'):
process_stream_message(str(incoming_valid_message['To']), # need to insert str() or mypy throws type error
incoming_valid_message,
debug_info)
except ZulipEmailForwardError as e:
# empty body throws exception
exception_message = str(e)
self.assertEqual(exception_message, "Unable to find plaintext or HTML message body")
def test_receive_stream_email_messages_empty_body(self) -> None:
# build dummy messages for stream
# test message with empty body is not sent
user_profile = self.example_user('hamlet')
self.login(user_profile.email)
self.subscribe(user_profile, "Denmark")
stream = get_stream("Denmark", user_profile.realm)
stream_to_address = encode_email_address(stream)
headers = {}
headers['Reply-To'] = self.example_email('othello')
# empty body
incoming_valid_message = MIMEText('')
incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
incoming_valid_message['From'] = self.example_email('hamlet')
incoming_valid_message['To'] = stream_to_address
incoming_valid_message['Reply-to'] = self.example_email('othello')
exception_message = ""
debug_info = {} # type: Dict[str, Any]
# process_message eats the exception & logs an error which can't be parsed here
# so calling process_stream_message directly
try:
process_stream_message(str(incoming_valid_message['To']), # need to insert str() or mypy throws type error
incoming_valid_message,
debug_info)
except ZulipEmailForwardError as e:
# empty body throws exception
exception_message = str(e)
self.assertEqual(exception_message, "Email has no nonempty body sections; ignoring.")
def test_remove_muted_topic(self) -> None:
user = self.example_user('hamlet')
email = user.email
realm = user.realm
self.login(email)
stream = get_stream(u'Verona', realm)
recipient = get_stream_recipient(stream.id)
url = '/api/v1/users/me/subscriptions/muted_topics'
payloads = [
{'stream': stream.name, 'topic': 'vERONA3', 'op': 'remove'},
{'stream_id': stream.id, 'topic': 'vEroNA3', 'op': 'remove'},
]
for data in payloads:
add_topic_mute(
user_profile=user,
stream_id=stream.id,
recipient_id=recipient.id,
topic_name='Verona3',
)
self.assertIn([stream.name, 'Verona3'], get_topic_mutes(user))
result = self.api_patch(email, url, data)
self.assert_json_success(result)
self.assertNotIn([stream.name, 'Verona3'], get_topic_mutes(user))
self.assertFalse(topic_is_muted(user, stream.id, 'verona3'))
def json_invite_users(request, user_profile, invitee_emails_raw=REQ("invitee_emails")):
# type: (HttpRequest, UserProfile, str) -> HttpResponse
if not invitee_emails_raw:
return json_error(_("You must specify at least one email address."))
invitee_emails = get_invitee_emails_set(invitee_emails_raw)
stream_names = request.POST.getlist('stream')
if not stream_names:
return json_error(_("You must specify at least one stream for invitees to join."))
# We unconditionally sub you to the notifications stream if it
# exists and is public.
notifications_stream = user_profile.realm.notifications_stream
if notifications_stream and not notifications_stream.invite_only:
stream_names.append(notifications_stream.name)
streams = [] # type: List[Stream]
for stream_name in stream_names:
stream = get_stream(stream_name, user_profile.realm)
if stream is None:
return json_error(_("Stream does not exist: %s. No invites were sent.") % (stream_name,))
streams.append(stream)
ret_error, error_data = do_invite_users(user_profile, invitee_emails, streams)
if ret_error is not None:
return json_error(data=error_data, msg=ret_error)
else:
return json_success()
def test_receive_stream_email_messages_empty_body(self):
# build dummy messages for stream
# test message with empty body is not sent
self.login("*****@*****.**")
user_profile = get_user_profile_by_email("*****@*****.**")
self.subscribe_to_stream(user_profile.email, "Denmark")
stream = get_stream("Denmark", user_profile.realm)
stream_to_address = encode_email_address(stream)
headers = {}
headers['Reply-To'] = '*****@*****.**'
# empty body
incoming_valid_message = MIMEText('')
incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
incoming_valid_message['From'] = "*****@*****.**"
incoming_valid_message['To'] = stream_to_address
incoming_valid_message['Reply-to'] = "*****@*****.**"
exception_message = ""
debug_info = {}
# process_message eats the exception & logs an error which can't be parsed here
# so calling process_stream_message directly
try:
process_stream_message(incoming_valid_message['To'],
incoming_valid_message['Subject'],
incoming_valid_message,
debug_info)
except ZulipEmailForwardError as e:
# empty body throws exception
exception_message = e.message
self.assertEqual(exception_message, "Unable to find plaintext or HTML message body")
def can_access_stream_history_by_name(user_profile: UserProfile, stream_name: str) -> bool:
"""Determine whether the provided user is allowed to access the
history of the target stream. The stream is specified by name.
This is used by the caller to determine whether this user can get
historical messages before they joined for a narrowing search.
Because of the way our search is currently structured,
we may be passed an invalid stream here. We return
False in that situation, and subsequent code will do
validation and raise the appropriate JsonableError.
Note that this function should only be used in contexts where
access_stream is being called elsewhere to confirm that the user
can actually see this stream.
"""
try:
stream = get_stream(stream_name, user_profile.realm)
except Stream.DoesNotExist:
return False
if stream.is_history_realm_public() and not user_profile.is_guest:
return True
if stream.is_history_public_to_subscribers():
# In this case, we check if the user is subscribed.
error = _("Invalid stream name '%s'" % (stream_name,))
try:
(recipient, sub) = access_stream_common(user_profile, stream, error)
except JsonableError:
return False
return True
return False
def subscribe_to_stream(self, email, stream_name, realm=None):
realm = get_realm(resolve_email_to_domain(email))
stream = get_stream(stream_name, realm)
if stream is None:
stream, _ = create_stream_if_needed(realm, stream_name)
user_profile = get_user_profile_by_email(email)
do_add_subscription(user_profile, stream, no_log=True)
def by_stream(self, query, operand, maybe_negate):
# type: (Query, str, ConditionTransform) -> Query
stream = get_stream(operand, self.user_profile.realm)
if stream is None:
raise BadNarrowOperator('unknown stream ' + operand)
if self.user_profile.realm.is_zephyr_mirror_realm:
# MIT users expect narrowing to "social" to also show messages to /^(un)*social(.d)*$/
# (unsocial, ununsocial, social.d, etc)
m = re.search(r'^(?:un)*(.+?)(?:\.d)*$', stream.name, re.IGNORECASE)
# Since the regex has a `.+` in it and "" is invalid as a
# stream name, this will always match
assert(m is not None)
base_stream_name = m.group(1)
matching_streams = get_active_streams(self.user_profile.realm).filter(
name__iregex=r'^(un)*%s(\.d)*$' % (self._pg_re_escape(base_stream_name),))
matching_stream_ids = [matching_stream.id for matching_stream in matching_streams]
recipients_map = bulk_get_recipients(Recipient.STREAM, matching_stream_ids)
cond = column("recipient_id").in_([recipient.id for recipient in recipients_map.values()])
return query.where(maybe_negate(cond))
recipient = get_recipient(Recipient.STREAM, type_id=stream.id)
cond = column("recipient_id") == recipient.id
return query.where(maybe_negate(cond))
def is_public_stream(stream, realm):
if not valid_stream_name(stream):
raise JsonableError("Invalid stream name")
stream = get_stream(stream, realm)
if stream is None:
return False
return stream.is_public()
def test_receive_stream_email_multiple_recipient_success(self) -> None:
user_profile = self.example_user('hamlet')
self.login(user_profile.email)
self.subscribe(user_profile, "Denmark")
stream = get_stream("Denmark", user_profile.realm)
# stream address is angle-addr within multiple addresses
stream_to_addresses = ["A.N. Other <*****@*****.**>",
"Denmark <{}>".format(encode_email_address(stream))]
incoming_valid_message = MIMEText('TestStreamEmailMessages Body')
incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
incoming_valid_message['From'] = self.example_email('hamlet')
incoming_valid_message['To'] = ", ".join(stream_to_addresses)
incoming_valid_message['Reply-to'] = self.example_email('othello')
process_message(incoming_valid_message)
# Hamlet is subscribed to this stream so should see the email message from Othello.
message = most_recent_message(user_profile)
self.assertEqual(message.content, "TestStreamEmailMessages Body")
self.assertEqual(get_display_recipient(message.recipient), stream.name)
self.assertEqual(message.topic_name(), incoming_valid_message['Subject'])
def test_error_no_recipient(self) -> None:
script = os.path.join(os.path.dirname(__file__),
'../../scripts/lib/email-mirror-postfix')
sender = self.example_email('hamlet')
stream = get_stream("Denmark", get_realm("zulip"))
stream_to_address = encode_email_address(stream)
template_path = os.path.join(MAILS_DIR, "simple.txt")
with open(template_path) as template_file:
mail_template = template_file.read()
mail = mail_template.format(stream_to_address=stream_to_address, sender=sender)
read_pipe, write_pipe = os.pipe()
os.write(write_pipe, mail.encode())
os.close(write_pipe)
success_call = True
try:
subprocess.check_output([script, '-s', settings.SHARED_SECRET, '-t'],
stdin=read_pipe)
except subprocess.CalledProcessError as e:
self.assertEqual(
e.output,
b'5.1.1 Bad destination mailbox address: No missed message email address.\n'
)
self.assertEqual(e.returncode, 67)
success_call = False
self.assertFalse(success_call)
def test_receive_stream_email_messages_success(self):
# build dummy messages for stream
# test valid incoming stream message is processed properly
self.login("*****@*****.**")
user_profile = get_user_profile_by_email("*****@*****.**")
self.subscribe_to_stream(user_profile.email, "Denmark")
stream = get_stream("Denmark", user_profile.realm)
stream_to_address = encode_email_address(stream)
incoming_valid_message = MIMEText('TestStreamEmailMessages Body')
incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
incoming_valid_message['From'] = "*****@*****.**"
incoming_valid_message['To'] = stream_to_address
incoming_valid_message['Reply-to'] = "*****@*****.**"
process_message(incoming_valid_message)
# Hamlet is subscribed to this stream so should see the email message from Othello.
message = most_recent_message(user_profile)
self.assertEqual(message.content, "TestStreamEmailMessages Body")
self.assertEqual(get_display_recipient(message.recipient), stream.name)
self.assertEqual(message.subject, incoming_valid_message['Subject'])
def test_receive_stream_email_messages_empty_body(self) -> None:
# build dummy messages for stream
# test message with empty body is not sent
user_profile = self.example_user('hamlet')
self.login(user_profile.email)
self.subscribe(user_profile, "Denmark")
stream = get_stream("Denmark", user_profile.realm)
stream_to_address = encode_email_address(stream)
headers = {}
headers['Reply-To'] = self.example_email('othello')
# empty body
incoming_valid_message = MIMEText('') # type: Any # https://github.com/python/typeshed/issues/275
incoming_valid_message['Subject'] = 'TestStreamEmailMessages Subject'
incoming_valid_message['From'] = self.example_email('hamlet')
incoming_valid_message['To'] = stream_to_address
incoming_valid_message['Reply-to'] = self.example_email('othello')
exception_message = ""
debug_info = {} # type: Dict[str, Any]
# process_message eats the exception & logs an error which can't be parsed here
# so calling process_stream_message directly
try:
process_stream_message(incoming_valid_message['To'],
incoming_valid_message['Subject'],
incoming_valid_message,
debug_info)
except ZulipEmailForwardError as e:
# empty body throws exception
exception_message = str(e)
self.assertEqual(exception_message, "Unable to find plaintext or HTML message body")
def subscribe(self, user_profile: UserProfile, stream_name: str) -> Stream:
try:
stream = get_stream(stream_name, user_profile.realm)
from_stream_creation = False
except Stream.DoesNotExist:
stream, from_stream_creation = create_stream_if_needed(user_profile.realm, stream_name)
bulk_add_subscriptions([stream], [user_profile], from_stream_creation=from_stream_creation)
return stream
def test_mark_all_in_invalid_topic_read(self) -> None:
self.login(self.example_email("hamlet"))
invalid_topic_name = "abc"
result = self.client_post("/json/mark_topic_as_read", {
"stream_id": get_stream("Denmark", get_realm("zulip")).id,
"topic_name": invalid_topic_name,
})
self.assert_json_error(result, 'No such topic \'abc\'')
def get_subscribers_backend(request, user_profile, stream_name=REQ('stream')):
stream = get_stream(stream_name, user_profile.realm)
if stream is None:
raise JsonableError("Stream does not exist: %s" % (stream_name,))
subscribers = get_subscriber_emails(stream, user_profile)
return json_success({'subscribers': subscribers})
def stream_or_none(stream_name, realm):
if stream_name == '':
return None
else:
stream = get_stream(stream_name, realm)
if not stream:
raise JsonableError('No such stream \'%s\'' % (stream_name,))
return stream
def handle(self, *args: Any, **options: str) -> None:
realm = self.get_realm(options)
assert realm is not None # Should be ensured by parser
old_name = options['old_name']
new_name = options['new_name']
stream = get_stream(old_name, realm)
do_rename_stream(stream, new_name)
def update_message_flags(request, user_profile,
messages=REQ(validator=check_list(check_int)),
operation=REQ('op'), flag=REQ(),
all=REQ(validator=check_bool, default=False),
stream_name=REQ(default=None),
topic_name=REQ(default=None)):
request._log_data["extra"] = "[%s %s]" % (operation, flag)
stream = None
if stream_name is not None:
stream = get_stream(stream_name, user_profile.realm)
if not stream:
raise JsonableError(_('No such stream \'%s\'') % (stream_name,))
if topic_name:
topic_exists = UserMessage.objects.filter(user_profile=user_profile,
message__recipient__type_id=stream.id,
message__recipient__type=Recipient.STREAM,
message__subject__iexact=topic_name).exists()
if not topic_exists:
raise JsonableError(_('No such topic \'%s\'') % (topic_name,))
do_update_message_flags(user_profile, operation, flag, messages, all, stream, topic_name)
return json_success({'result': 'success',
'messages': messages,
'msg': ''})
def test_exclude_subscription_modified_streams(self) -> None:
othello = self.example_user('othello')
for stream in ['Verona', 'Scotland', 'Denmark']:
self.subscribe(othello, stream)
# Delete all RealmAuditLogs to ignore any changes to subscriptions to
# streams done for the setup.
RealmAuditLog.objects.all().delete()
realm = othello.realm
stream_names = self.get_streams(othello.email, realm)
stream_ids = {name: get_stream(name, realm).id for name in stream_names}
# Unsubscribe and subscribe from a stream
self.unsubscribe(othello, 'Verona')
self.subscribe(othello, 'Verona')
one_sec_ago = timezone_now() - datetime.timedelta(seconds=1)
filtered_stream_ids = exclude_subscription_modified_streams(
othello, list(stream_ids.values()), one_sec_ago)
self.assertNotIn(stream_ids['Verona'], filtered_stream_ids)
self.assertIn(stream_ids['Scotland'], filtered_stream_ids)
self.assertIn(stream_ids['Denmark'], filtered_stream_ids)
def json_invite_users(request,
user_profile,
invitee_emails_raw=REQ("invitee_emails")):
# type: (HttpRequest, UserProfile, str) -> HttpResponse
if not invitee_emails_raw:
return json_error(_("You must specify at least one email address."))
invitee_emails = get_invitee_emails_set(invitee_emails_raw)
stream_names = request.POST.getlist('stream')
if not stream_names:
return json_error(
_("You must specify at least one stream for invitees to join."))
# We unconditionally sub you to the notifications stream if it
# exists and is public.
notifications_stream = user_profile.realm.notifications_stream
if notifications_stream and not notifications_stream.invite_only:
stream_names.append(notifications_stream.name)
streams = [] # type: List[Stream]
for stream_name in stream_names:
stream = get_stream(stream_name, user_profile.realm)
if stream is None:
return json_error(
_("Stream does not exist: %s. No invites were sent.") %
(stream_name, ))
streams.append(stream)
ret_error, error_data = do_invite_users(user_profile, invitee_emails,
streams)
if ret_error is not None:
return json_error(data=error_data, msg=ret_error)
else:
return json_success()
def test_muted_topic_remove_invalid(self) -> None:
user = self.example_user('hamlet')
realm = user.realm
self.login_user(user)
stream = get_stream('Verona', realm)
url = '/api/v1/users/me/subscriptions/muted_topics'
data: Dict[str, Any] = {
'stream': 'BOGUS',
'topic': 'Verona3',
'op': 'remove'
}
result = self.api_patch(user, url, data)
self.assert_json_error(result, "Topic is not muted")
data = {'stream': stream.name, 'topic': 'BOGUS', 'op': 'remove'}
result = self.api_patch(user, url, data)
self.assert_json_error(result, "Topic is not muted")
data = {'stream_id': 999999999, 'topic': 'BOGUS', 'op': 'remove'}
result = self.api_patch(user, url, data)
self.assert_json_error(result, "Topic is not muted")
data = {'topic': 'Verona3', 'op': 'remove'}
result = self.api_patch(user, url, data)
self.assert_json_error(result, "Please supply 'stream'.")
data = {
'stream': stream.name,
'stream_id': stream.id,
'topic': 'Verona3',
'op': 'remove'
}
result = self.api_patch(user, url, data)
self.assert_json_error(result,
"Please choose one: 'stream' or 'stream_id'.")
def by_stream(self, query, operand, maybe_negate):
stream = get_stream(operand, self.user_profile.realm)
if stream is None:
raise BadNarrowOperator('unknown stream ' + operand)
if self.user_profile.realm.domain == "mit.edu":
# MIT users expect narrowing to "social" to also show messages to /^(un)*social(.d)*$/
# (unsocial, ununsocial, social.d, etc)
m = re.search(r'^(?:un)*(.+?)(?:\.d)*$', stream.name, re.IGNORECASE)
if m:
base_stream_name = m.group(1)
else:
base_stream_name = stream.name
matching_streams = get_active_streams(self.user_profile.realm).filter(
name__iregex=r'^(un)*%s(\.d)*$' % (self._pg_re_escape(base_stream_name),))
matching_stream_ids = [matching_stream.id for matching_stream in matching_streams]
recipients_map = bulk_get_recipients(Recipient.STREAM, matching_stream_ids)
cond = column("recipient_id").in_([recipient.id for recipient in recipients_map.values()])
return query.where(maybe_negate(cond))
recipient = get_recipient(Recipient.STREAM, type_id=stream.id)
cond = column("recipient_id") == recipient.id
return query.where(maybe_negate(cond))
def test_muted_topic_remove_invalid(self) -> None:
user = self.example_user("hamlet")
realm = user.realm
self.login_user(user)
stream = get_stream("Verona", realm)
url = "/api/v1/users/me/subscriptions/muted_topics"
data: Dict[str, Any] = {
"stream": "BOGUS",
"topic": "Verona3",
"op": "remove"
}
result = self.api_patch(user, url, data)
self.assert_json_error(result, "Topic is not muted")
data = {"stream": stream.name, "topic": "BOGUS", "op": "remove"}
result = self.api_patch(user, url, data)
self.assert_json_error(result, "Topic is not muted")
data = {"stream_id": 999999999, "topic": "BOGUS", "op": "remove"}
result = self.api_patch(user, url, data)
self.assert_json_error(result, "Topic is not muted")
data = {"topic": "Verona3", "op": "remove"}
result = self.api_patch(user, url, data)
self.assert_json_error(result, "Please supply 'stream'.")
data = {
"stream": stream.name,
"stream_id": stream.id,
"topic": "Verona3",
"op": "remove"
}
result = self.api_patch(user, url, data)
self.assert_json_error(result,
"Please choose one: 'stream' or 'stream_id'.")
def test_get_deactivated_muted_topic(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
stream = get_stream("Verona", user.realm)
recipient = stream.recipient
mock_date_muted = datetime(2020, 1, 1, tzinfo=timezone.utc).timestamp()
add_topic_mute(
user_profile=user,
stream_id=stream.id,
recipient_id=recipient.id,
topic_name="Verona3",
date_muted=datetime(2020, 1, 1, tzinfo=timezone.utc),
)
stream.deactivated = True
stream.save()
self.assertNotIn((stream.name, "Verona3", mock_date_muted),
get_topic_mutes(user))
self.assertIn((stream.name, "Verona3", mock_date_muted),
get_topic_mutes(user, True))
def test_mirror_worker_rate_limiting(self, mock_mirror_email: MagicMock,
mock_warn: MagicMock) -> None:
fake_client = self.FakeClient()
realm = get_realm('zulip')
RateLimitedRealmMirror(realm).clear_history()
stream = get_stream('Denmark', realm)
stream_to_address = encode_email_address(stream)
data = [
dict(
msg_base64=base64.b64encode(b'\xf3test').decode(),
time=time.time(),
rcpt_to=stream_to_address,
),
] * 5
for element in data:
fake_client.queue.append(('email_mirror', element))
with simulated_queue_client(lambda: fake_client), \
self.assertLogs('zerver.worker.queue_processors', level='WARNING') as warn_logs:
start_time = time.time()
with patch('time.time', return_value=start_time):
worker = queue_processors.MirrorWorker()
worker.setup()
worker.start()
# Of the first 5 messages, only 2 should be processed
# (the rest being rate-limited):
self.assertEqual(mock_mirror_email.call_count, 2)
# If a new message is sent into the stream mirror, it will get rejected:
fake_client.queue.append(('email_mirror', data[0]))
worker.start()
self.assertEqual(mock_mirror_email.call_count, 2)
# However, missed message emails don't get rate limited:
with self.settings(EMAIL_GATEWAY_PATTERN="*****@*****.**"):
address = 'mm' + ('x' * 32) + '@example.com'
event = dict(
msg_base64=base64.b64encode(b'\xf3test').decode(),
time=time.time(),
rcpt_to=address,
)
fake_client.queue.append(('email_mirror', event))
worker.start()
self.assertEqual(mock_mirror_email.call_count, 3)
# After some times passes, emails get accepted again:
with patch('time.time', return_value=(start_time + 11.0)):
fake_client.queue.append(('email_mirror', data[0]))
worker.start()
self.assertEqual(mock_mirror_email.call_count, 4)
# If RateLimiterLockingException is thrown, we rate-limit the new message:
with patch(
'zerver.lib.rate_limiter.RedisRateLimiterBackend.incr_ratelimit',
side_effect=RateLimiterLockingException):
fake_client.queue.append(('email_mirror', data[0]))
worker.start()
self.assertEqual(mock_mirror_email.call_count, 4)
mock_warn.assert_called_with(
"Deadlock trying to incr_ratelimit for %s",
f"RateLimitedRealmMirror:{realm.string_id}",
)
self.assertEqual(warn_logs.output, [
'WARNING:zerver.worker.queue_processors:MirrorWorker: Rejecting an email from: None to realm: Zulip Dev - rate limited.'
] * 5)
def test_end_to_end_missedmessage_hook(self) -> None:
"""Tests what arguments missedmessage_hook passes into maybe_enqueue_notifications.
Combined with the previous test, this ensures that the missedmessage_hook is correct"""
user_profile = self.example_user("hamlet")
cordelia = self.example_user("cordelia")
user_profile.enable_online_push_notifications = False
user_profile.save()
iago = self.example_user("iago")
# Fetch the Denmark stream for testing
stream = get_stream("Denmark", user_profile.realm)
sub = Subscription.objects.get(
user_profile=user_profile,
recipient__type=Recipient.STREAM,
recipient__type_id=stream.id,
)
self.login_user(user_profile)
def change_subscription_properties(
user_profile: UserProfile,
stream: Stream,
sub: Subscription,
properties: Dict[str, bool],
) -> None:
for property_name, value in properties.items():
do_change_subscription_property(
user_profile, sub, stream, property_name, value, acting_user=None
)
def allocate_event_queue() -> ClientDescriptor:
result = self.tornado_call(
get_events,
user_profile,
{
"apply_markdown": orjson.dumps(True).decode(),
"client_gravatar": orjson.dumps(True).decode(),
"event_types": orjson.dumps(["message"]).decode(),
"user_client": "website",
"dont_block": orjson.dumps(True).decode(),
},
)
self.assert_json_success(result)
queue_id = orjson.loads(result.content)["queue_id"]
return get_client_descriptor(queue_id)
def destroy_event_queue(queue_id: str) -> None:
result = self.tornado_call(cleanup_event_queue, user_profile, {"queue_id": queue_id})
self.assert_json_success(result)
def assert_maybe_enqueue_notifications_call_args(
args_dict: Collection[Any],
message_id: int,
**kwargs: Any,
) -> None:
expected_args_dict = self.get_maybe_enqueue_notifications_parameters(
user_id=user_profile.id,
acting_user_id=iago.id,
message_id=message_id,
**kwargs,
)
self.assertEqual(args_dict, expected_args_dict)
client_descriptor = allocate_event_queue()
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
# To test the missed_message hook, we first need to send a message
msg_id = self.send_stream_message(iago, "Denmark")
# Verify that nothing happens if you call it as not the
# "last client descriptor", in which case the function
# short-circuits, since the `missedmessage_hook` handler
# for garbage-collection is only for the user's last queue.
missedmessage_hook(user_profile.id, client_descriptor, False)
mock_enqueue.assert_not_called()
# Now verify that we called the appropriate enqueue function
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
already_notified={"email_notified": False, "push_notified": False},
)
destroy_event_queue(client_descriptor.event_queue.id)
# Test the hook with a private message; this should trigger notifications
client_descriptor = allocate_event_queue()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_personal_message(iago, user_profile)
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
pm_email_notify=True,
pm_push_notify=True,
already_notified={"email_notified": True, "push_notified": True},
)
destroy_event_queue(client_descriptor.event_queue.id)
# If `enable_offline_email_notifications` is disabled, email otifications shouldn't
# be sent even for PMs
user_profile.enable_offline_email_notifications = False
user_profile.save()
client_descriptor = allocate_event_queue()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_personal_message(iago, user_profile)
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
pm_email_notify=False,
pm_push_notify=True,
already_notified={"email_notified": False, "push_notified": True},
)
destroy_event_queue(client_descriptor.event_queue.id)
user_profile.enable_offline_email_notifications = True
user_profile.save()
# Test the hook with a mention; this should trigger notifications
client_descriptor = allocate_event_queue()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_stream_message(
self.example_user("iago"), "Denmark", content="@**King Hamlet** what's up?"
)
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
mention_push_notify=True,
mention_email_notify=True,
already_notified={"email_notified": True, "push_notified": True},
)
destroy_event_queue(client_descriptor.event_queue.id)
# If `enable_offline_push_notifications` is disabled, push otifications shouldn't
# be sent even for mentions
user_profile.enable_offline_push_notifications = False
user_profile.save()
client_descriptor = allocate_event_queue()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_personal_message(iago, user_profile)
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
pm_email_notify=True,
pm_push_notify=False,
already_notified={"email_notified": True, "push_notified": False},
)
destroy_event_queue(client_descriptor.event_queue.id)
user_profile.enable_offline_push_notifications = True
user_profile.save()
# Test the hook with a wildcard mention; this should trigger notifications
client_descriptor = allocate_event_queue()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_stream_message(iago, "Denmark", content="@**all** what's up?")
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
wildcard_mention_email_notify=True,
wildcard_mention_push_notify=True,
already_notified={"email_notified": True, "push_notified": True},
)
destroy_event_queue(client_descriptor.event_queue.id)
# Wildcard mentions in muted streams don't notify.
change_subscription_properties(user_profile, stream, sub, {"is_muted": True})
client_descriptor = allocate_event_queue()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_stream_message(iago, "Denmark", content="@**all** what's up?")
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
wildcard_mention_email_notify=False,
wildcard_mention_push_notify=False,
message_id=msg_id,
already_notified={"email_notified": False, "push_notified": False},
)
destroy_event_queue(client_descriptor.event_queue.id)
change_subscription_properties(user_profile, stream, sub, {"is_muted": False})
# With wildcard_mentions_notify=False, we treat the user as not mentioned.
user_profile.wildcard_mentions_notify = False
user_profile.save()
client_descriptor = allocate_event_queue()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_stream_message(iago, "Denmark", content="@**all** what's up?")
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
wildcard_mention_email_notify=False,
wildcard_mention_push_notify=False,
already_notified={"email_notified": False, "push_notified": False},
)
destroy_event_queue(client_descriptor.event_queue.id)
user_profile.wildcard_mentions_notify = True
user_profile.save()
# If wildcard_mentions_notify=True for a stream and False for a user, we treat the user
# as mentioned for that stream.
user_profile.wildcard_mentions_notify = False
sub.wildcard_mentions_notify = True
user_profile.save()
sub.save()
client_descriptor = allocate_event_queue()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_stream_message(iago, "Denmark", content="@**all** what's up?")
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
wildcard_mention_email_notify=True,
wildcard_mention_push_notify=True,
already_notified={"email_notified": True, "push_notified": True},
)
destroy_event_queue(client_descriptor.event_queue.id)
user_profile.wildcard_mentions_notify = True
sub.wildcard_mentions_notify = None
user_profile.save()
sub.save()
# If notifications for personal mentions themselves have been turned off,
# even turning on `wildcard_mentions_notify` should not send notifications
user_profile.enable_offline_email_notifications = False
user_profile.wildcard_mentions_notify = True
user_profile.save()
client_descriptor = allocate_event_queue()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_stream_message(iago, "Denmark", content="@**all** what's up?")
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
# We've turned off email notifications for personal mentions, but push notifications
# for personal mentions are still on.
# Because `wildcard_mentions_notify` is True, a message with `@all` should follow the
# personal mention settings
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
wildcard_mention_email_notify=False,
wildcard_mention_push_notify=True,
already_notified={"email_notified": False, "push_notified": True},
)
destroy_event_queue(client_descriptor.event_queue.id)
user_profile.enable_offline_email_notifications = True
user_profile.wildcard_mentions_notify = True
user_profile.save()
# Test with a user group mention
hamlet_and_cordelia = create_user_group(
"hamlet_and_cordelia", [user_profile, cordelia], cordelia.realm
)
client_descriptor = allocate_event_queue()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_stream_message(
iago, "Denmark", content="@*hamlet_and_cordelia* what's up?"
)
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
mention_push_notify=True,
mention_email_notify=True,
mentioned_user_group_id=hamlet_and_cordelia.id,
already_notified={"email_notified": True, "push_notified": True},
)
destroy_event_queue(client_descriptor.event_queue.id)
remove_user_from_user_group(user_profile, hamlet_and_cordelia)
remove_user_from_user_group(cordelia, hamlet_and_cordelia)
# Test the hook with a stream message with stream_push_notify
change_subscription_properties(user_profile, stream, sub, {"push_notifications": True})
client_descriptor = allocate_event_queue()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_stream_message(iago, "Denmark", content="what's up everyone?")
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
stream_push_notify=True,
stream_email_notify=False,
already_notified={"email_notified": False, "push_notified": True},
)
destroy_event_queue(client_descriptor.event_queue.id)
# Test the hook with a stream message with stream_email_notify
client_descriptor = allocate_event_queue()
change_subscription_properties(
user_profile, stream, sub, {"push_notifications": False, "email_notifications": True}
)
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_stream_message(iago, "Denmark", content="what's up everyone?")
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
stream_push_notify=False,
stream_email_notify=True,
already_notified={"email_notified": True, "push_notified": False},
)
destroy_event_queue(client_descriptor.event_queue.id)
# Test the hook with stream message with stream_push_notify on
# a muted topic, which we should not push notify for
client_descriptor = allocate_event_queue()
change_subscription_properties(
user_profile, stream, sub, {"push_notifications": True, "email_notifications": False}
)
self.assertTrue(client_descriptor.event_queue.empty())
do_mute_topic(user_profile, stream, "mutingtest")
msg_id = self.send_stream_message(
iago,
"Denmark",
content="what's up everyone?",
topic_name="mutingtest",
)
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
already_notified={"email_notified": False, "push_notified": False},
)
destroy_event_queue(client_descriptor.event_queue.id)
# Test the hook with stream message with stream_email_notify on
# a muted stream, which we should not push notify for
client_descriptor = allocate_event_queue()
change_subscription_properties(
user_profile, stream, sub, {"push_notifications": False, "email_notifications": True}
)
self.assertTrue(client_descriptor.event_queue.empty())
change_subscription_properties(user_profile, stream, sub, {"is_muted": True})
msg_id = self.send_stream_message(iago, "Denmark", content="what's up everyone?")
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
already_notified={"email_notified": False, "push_notified": False},
)
destroy_event_queue(client_descriptor.event_queue.id)
# Clean up the state we just changed (not necessary unless we add more test code below)
change_subscription_properties(
user_profile, stream, sub, {"push_notifications": True, "is_muted": False}
)
# Test the hook when the sender has been muted
result = self.api_post(user_profile, f"/api/v1/users/me/muted_users/{iago.id}")
self.assert_json_success(result)
client_descriptor = allocate_event_queue()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_personal_message(iago, user_profile)
with mock.patch("zerver.tornado.event_queue.maybe_enqueue_notifications") as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_dict = mock_enqueue.call_args_list[0][1]
assert_maybe_enqueue_notifications_call_args(
args_dict=args_dict,
message_id=msg_id,
sender_is_muted=True,
pm_email_notify=True,
pm_push_notify=True,
already_notified={"email_notified": False, "push_notified": False},
)
destroy_event_queue(client_descriptor.event_queue.id)
result = self.api_delete(user_profile, f"/api/v1/users/me/muted_users/{iago.id}")
self.assert_json_success(result)
def test_stream_recipient_info(self) -> None:
hamlet = self.example_user('hamlet')
cordelia = self.example_user('cordelia')
othello = self.example_user('othello')
realm = hamlet.realm
stream_name = 'Test Stream'
topic_name = 'test topic'
for user in [hamlet, cordelia, othello]:
self.subscribe(user, stream_name)
stream = get_stream(stream_name, realm)
recipient = get_stream_recipient(stream.id)
stream_topic = StreamTopicTarget(
stream_id=stream.id,
topic_name=topic_name,
)
sub = get_subscription(stream_name, hamlet)
sub.push_notifications = True
sub.save()
info = get_recipient_info(
recipient=recipient,
sender_id=hamlet.id,
stream_topic=stream_topic,
)
all_user_ids = {hamlet.id, cordelia.id, othello.id}
expected_info = dict(
active_user_ids=all_user_ids,
push_notify_user_ids=set(),
stream_push_user_ids={hamlet.id},
stream_email_user_ids=set(),
um_eligible_user_ids=all_user_ids,
long_term_idle_user_ids=set(),
default_bot_user_ids=set(),
service_bot_tuples=[],
)
self.assertEqual(info, expected_info)
# Now mute Hamlet to omit him from stream_push_user_ids.
add_topic_mute(
user_profile=hamlet,
stream_id=stream.id,
recipient_id=recipient.id,
topic_name=topic_name,
)
info = get_recipient_info(
recipient=recipient,
sender_id=hamlet.id,
stream_topic=stream_topic,
)
self.assertEqual(info['stream_push_user_ids'], set())
# Add a service bot.
service_bot = do_create_user(
email='*****@*****.**',
password='',
realm=realm,
full_name='',
short_name='',
bot_type=UserProfile.EMBEDDED_BOT,
)
info = get_recipient_info(recipient=recipient,
sender_id=hamlet.id,
stream_topic=stream_topic,
possibly_mentioned_user_ids={service_bot.id})
self.assertEqual(info['service_bot_tuples'], [
(service_bot.id, UserProfile.EMBEDDED_BOT),
])
# Add a normal bot.
normal_bot = do_create_user(
email='*****@*****.**',
password='',
realm=realm,
full_name='',
short_name='',
bot_type=UserProfile.DEFAULT_BOT,
)
info = get_recipient_info(
recipient=recipient,
sender_id=hamlet.id,
stream_topic=stream_topic,
possibly_mentioned_user_ids={service_bot.id, normal_bot.id})
self.assertEqual(info['default_bot_user_ids'], {normal_bot.id})
def unsubscribe_from_stream(self, email, stream_name):
# type: (Text, Text) -> None
user_profile = get_user_profile_by_email(email)
stream = get_stream(stream_name, user_profile.realm)
bulk_remove_subscriptions([user_profile], [stream])
def unsubscribe(self, user_profile: UserProfile, stream_name: str) -> None:
client = get_client("website")
stream = get_stream(stream_name, user_profile.realm)
bulk_remove_subscriptions([user_profile], [stream], client)
def setUp(self) -> None:
super().setUp()
self.user_profile = self.example_user("hamlet")
self.email = self.user_profile.email
self.stream = get_stream("Verona", self.user_profile.realm)
def test_success_stream(self) -> None:
stream = get_stream("Denmark", get_realm("zulip"))
stream_to_address = encode_email_address(stream)
result = self.send_offline_message(stream_to_address,
self.example_email('hamlet'))
self.assert_json_success(result)
def test_end_to_end_missedmessage_hook(self) -> None:
"""Tests what arguments missedmessage_hook passes into maybe_enqueue_notifications.
Combined with the previous test, this ensures that the missedmessage_hook is correct"""
user_profile = self.example_user('hamlet')
email = user_profile.email
self.login(email)
def change_subscription_properties(
user_profile: UserProfile, stream: Stream, sub: Subscription,
properties: Dict[str, bool]) -> None:
for property_name, value in properties.items():
do_change_subscription_property(user_profile, sub, stream,
property_name, value)
result = self.tornado_call(
get_events, user_profile, {
"apply_markdown": ujson.dumps(True),
"client_gravatar": ujson.dumps(True),
"event_types": ujson.dumps(["message"]),
"user_client": "website",
"dont_block": ujson.dumps(True),
})
self.assert_json_success(result)
queue_id = ujson.loads(result.content)["queue_id"]
client_descriptor = get_client_descriptor(queue_id)
with mock.patch(
"zerver.tornado.event_queue.maybe_enqueue_notifications"
) as mock_enqueue:
# To test the missed_message hook, we first need to send a message
msg_id = self.send_stream_message(self.example_email("iago"),
"Denmark")
# Verify that nothing happens if you call it as not the
# "last client descriptor", in which case the function
# short-circuits, since the `missedmessage_hook` handler
# for garbage-collection is only for the user's last queue.
missedmessage_hook(user_profile.id, client_descriptor, False)
mock_enqueue.assert_not_called()
# Now verify that we called the appropriate enqueue function
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_list = mock_enqueue.call_args_list[0][0]
self.assertEqual(args_list,
(user_profile.id, msg_id, False, False, False,
False, "Denmark", False, True, {
'email_notified': False,
'push_notified': False
}))
# Clear the event queue, before repeating with a private message
client_descriptor.event_queue.pop()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_personal_message(self.example_email("iago"), email)
with mock.patch(
"zerver.tornado.event_queue.maybe_enqueue_notifications"
) as mock_enqueue:
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_list = mock_enqueue.call_args_list[0][0]
self.assertEqual(args_list, (user_profile.id, msg_id, True, False,
False, False, None, False, True, {
'email_notified': True,
'push_notified': True
}))
# Clear the event queue, now repeat with a mention
client_descriptor.event_queue.pop()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_stream_message(
self.example_email("iago"),
"Denmark",
content="@**King Hamlet** what's up?")
with mock.patch(
"zerver.tornado.event_queue.maybe_enqueue_notifications"
) as mock_enqueue:
# Clear the event queue, before repeating with a private message
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_list = mock_enqueue.call_args_list[0][0]
self.assertEqual(args_list,
(user_profile.id, msg_id, False, True, False,
False, "Denmark", False, True, {
'email_notified': True,
'push_notified': True
}))
stream = get_stream("Denmark", user_profile.realm)
sub = Subscription.objects.get(user_profile=user_profile,
recipient__type=Recipient.STREAM,
recipient__type_id=stream.id)
# Clear the event queue, now repeat with stream message with stream_push_notify
change_subscription_properties(user_profile, stream, sub,
{'push_notifications': True})
client_descriptor.event_queue.pop()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_stream_message(self.example_email("iago"),
"Denmark",
content="what's up everyone?")
with mock.patch(
"zerver.tornado.event_queue.maybe_enqueue_notifications"
) as mock_enqueue:
# Clear the event queue, before repeating with a private message
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_list = mock_enqueue.call_args_list[0][0]
self.assertEqual(args_list, (user_profile.id, msg_id, False, False,
True, False, "Denmark", False, True, {
'email_notified': False,
'push_notified': False
}))
# Clear the event queue, now repeat with stream message with stream_email_notify
change_subscription_properties(user_profile, stream, sub, {
'push_notifications': False,
'email_notifications': True
})
client_descriptor.event_queue.pop()
self.assertTrue(client_descriptor.event_queue.empty())
msg_id = self.send_stream_message(self.example_email("iago"),
"Denmark",
content="what's up everyone?")
with mock.patch(
"zerver.tornado.event_queue.maybe_enqueue_notifications"
) as mock_enqueue:
# Clear the event queue, before repeating with a private message
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_list = mock_enqueue.call_args_list[0][0]
self.assertEqual(args_list, (user_profile.id, msg_id, False, False,
False, True, "Denmark", False, True, {
'email_notified': False,
'push_notified': False
}))
# Clear the event queue, now repeat with stream message with stream_push_notify
# on a muted topic, which we should not push notify for
change_subscription_properties(user_profile, stream, sub, {
'push_notifications': True,
'email_notifications': False
})
client_descriptor.event_queue.pop()
self.assertTrue(client_descriptor.event_queue.empty())
do_mute_topic(user_profile, stream, sub.recipient, "mutingtest")
msg_id = self.send_stream_message(self.example_email("iago"),
"Denmark",
content="what's up everyone?",
topic_name="mutingtest")
with mock.patch(
"zerver.tornado.event_queue.maybe_enqueue_notifications"
) as mock_enqueue:
# Clear the event queue, before repeating with a private message
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_list = mock_enqueue.call_args_list[0][0]
self.assertEqual(args_list,
(user_profile.id, msg_id, False, False, False,
False, "Denmark", False, True, {
'email_notified': False,
'push_notified': False
}))
# Clear the event queue, now repeat with stream message with stream_email_notify
# on a muted stream, which we should not email notify for
change_subscription_properties(user_profile, stream, sub, {
'push_notifications': False,
'email_notifications': True
})
client_descriptor.event_queue.pop()
self.assertTrue(client_descriptor.event_queue.empty())
change_subscription_properties(user_profile, stream, sub,
{'in_home_view': False})
msg_id = self.send_stream_message(self.example_email("iago"),
"Denmark",
content="what's up everyone?")
with mock.patch(
"zerver.tornado.event_queue.maybe_enqueue_notifications"
) as mock_enqueue:
# Clear the event queue, before repeating with a private message
missedmessage_hook(user_profile.id, client_descriptor, True)
mock_enqueue.assert_called_once()
args_list = mock_enqueue.call_args_list[0][0]
self.assertEqual(args_list,
(user_profile.id, msg_id, False, False, False,
False, "Denmark", False, True, {
'email_notified': False,
'push_notified': False
}))
# Clean up the state we just changed (not necessary unless we add more test code below)
change_subscription_properties(user_profile, stream, sub, {
'push_notifications': True,
'in_home_view': True
})
def test_mirror_worker_rate_limiting(self, mock_mirror_email: MagicMock) -> None:
fake_client = self.FakeClient()
realm = get_realm("zulip")
RateLimitedRealmMirror(realm).clear_history()
stream = get_stream("Denmark", realm)
stream_to_address = encode_email_address(stream)
data = [
dict(
msg_base64=base64.b64encode(b"\xf3test").decode(),
time=time.time(),
rcpt_to=stream_to_address,
),
] * 5
for element in data:
fake_client.enqueue("email_mirror", element)
with simulated_queue_client(lambda: fake_client), self.assertLogs(
"zerver.worker.queue_processors", level="WARNING"
) as warn_logs:
start_time = time.time()
with patch("time.time", return_value=start_time):
worker = queue_processors.MirrorWorker()
worker.setup()
worker.start()
# Of the first 5 messages, only 2 should be processed
# (the rest being rate-limited):
self.assertEqual(mock_mirror_email.call_count, 2)
# If a new message is sent into the stream mirror, it will get rejected:
fake_client.enqueue("email_mirror", data[0])
worker.start()
self.assertEqual(mock_mirror_email.call_count, 2)
# However, message notification emails don't get rate limited:
with self.settings(EMAIL_GATEWAY_PATTERN="*****@*****.**"):
address = "mm" + ("x" * 32) + "@example.com"
event = dict(
msg_base64=base64.b64encode(b"\xf3test").decode(),
time=time.time(),
rcpt_to=address,
)
fake_client.enqueue("email_mirror", event)
worker.start()
self.assertEqual(mock_mirror_email.call_count, 3)
# After some times passes, emails get accepted again:
with patch("time.time", return_value=(start_time + 11.0)):
fake_client.enqueue("email_mirror", data[0])
worker.start()
self.assertEqual(mock_mirror_email.call_count, 4)
# If RateLimiterLockingException is thrown, we rate-limit the new message:
with patch(
"zerver.lib.rate_limiter.RedisRateLimiterBackend.incr_ratelimit",
side_effect=RateLimiterLockingException,
):
with self.assertLogs("zerver.lib.rate_limiter", "WARNING") as mock_warn:
fake_client.enqueue("email_mirror", data[0])
worker.start()
self.assertEqual(mock_mirror_email.call_count, 4)
self.assertEqual(
mock_warn.output,
[
"WARNING:zerver.lib.rate_limiter:Deadlock trying to incr_ratelimit for RateLimitedRealmMirror:zulip"
],
)
self.assertEqual(
warn_logs.output,
[
"WARNING:zerver.worker.queue_processors:MirrorWorker: Rejecting an email from: None to realm: Zulip Dev - rate limited."
]
* 5,
)
def test_encode_email_addr(self) -> None:
stream = get_stream("Denmark", get_realm("zulip"))
with self.settings(EMAIL_GATEWAY_PATTERN=''):
test_address = encode_email_address(stream)
self.assertEqual(test_address, '')
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write(
"Error! More than 100% of messages allocated.\n")
return
# Get consistent data for backend tests.
if options["test_suite"]:
random.seed(0)
with connection.cursor() as cursor:
# Sometimes bugs relating to confusing recipient.id for recipient.type_id
# or <object>.id for <object>.recipient_id remain undiscovered by the test suite
# due to these numbers happening to coincide in such a way that it makes tests
# accidentally pass. By bumping the Recipient.id sequence by a large enough number,
# we can have those ids in a completely different range of values than object ids,
# eliminatng the possibility of such coincidences.
cursor.execute("SELECT setval('zerver_recipient_id_seq', 100)")
# If max_topics is not set, we set it proportional to the
# number of messages.
if options["max_topics"] is None:
options["max_topics"] = 1 + options["num_messages"] // 100
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our three default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
create_internal_realm()
zulip_realm = do_create_realm(
string_id="zulip",
name="Zulip Dev",
emails_restricted_to_domains=False,
email_address_visibility=Realm.EMAIL_ADDRESS_VISIBILITY_ADMINS,
description=
"The Zulip development environment default organization."
" It's great for testing!",
invite_required=False,
plan_type=Realm.SELF_HOSTED,
org_type=Realm.ORG_TYPES["business"]["id"],
)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
assert zulip_realm.notifications_stream is not None
zulip_realm.notifications_stream.name = "Verona"
zulip_realm.notifications_stream.description = "A city in Italy"
zulip_realm.notifications_stream.save(
update_fields=["name", "description"])
if options["test_suite"]:
mit_realm = do_create_realm(
string_id="zephyr",
name="MIT",
emails_restricted_to_domains=True,
invite_required=False,
plan_type=Realm.SELF_HOSTED,
org_type=Realm.ORG_TYPES["business"]["id"],
)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = do_create_realm(
string_id="lear",
name="Lear & Co.",
emails_restricted_to_domains=False,
invite_required=False,
plan_type=Realm.SELF_HOSTED,
org_type=Realm.ORG_TYPES["business"]["id"],
)
# Default to allowing all members to send mentions in
# large streams for the test suite to keep
# mention-related tests simple.
zulip_realm.wildcard_mention_policy = Realm.WILDCARD_MENTION_POLICY_MEMBERS
zulip_realm.save(update_fields=["wildcard_mention_policy"])
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia, Lear's daughter", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
("Polonius", "*****@*****.**"),
("Desdemona", "*****@*****.**"),
("शिव", "*****@*****.**"),
]
# For testing really large batches:
# Create extra users with semi realistic names to make search
# functions somewhat realistic. We'll still create 1000 users
# like Extra222 User for some predicability.
num_names = options["extra_users"]
num_boring_names = 300
for i in range(min(num_names, num_boring_names)):
full_name = f"Extra{i:03} User"
names.append((full_name, f"extrauser{i}@zulip.com"))
if num_names > num_boring_names:
fnames = [
"Amber",
"Arpita",
"Bob",
"Cindy",
"Daniela",
"Dan",
"Dinesh",
"Faye",
"François",
"George",
"Hank",
"Irene",
"James",
"Janice",
"Jenny",
"Jill",
"John",
"Kate",
"Katelyn",
"Kobe",
"Lexi",
"Manish",
"Mark",
"Matt",
"Mayna",
"Michael",
"Pete",
"Peter",
"Phil",
"Phillipa",
"Preston",
"Sally",
"Scott",
"Sandra",
"Steve",
"Stephanie",
"Vera",
]
mnames = ["de", "van", "von", "Shaw", "T."]
lnames = [
"Adams",
"Agarwal",
"Beal",
"Benson",
"Bonita",
"Davis",
"George",
"Harden",
"James",
"Jones",
"Johnson",
"Jordan",
"Lee",
"Leonard",
"Singh",
"Smith",
"Patel",
"Towns",
"Wall",
]
non_ascii_names = [
"Günter",
"أحمد",
"Magnús",
"आशी",
"イツキ",
"语嫣",
"அருண்",
"Александр",
"José",
]
# to imitate emoji insertions in usernames
raw_emojis = ["😎", "😂", "🐱👤"]
for i in range(num_boring_names, num_names):
fname = random.choice(fnames) + str(i)
full_name = fname
if random.random() < 0.7:
if random.random() < 0.3:
full_name += " " + random.choice(non_ascii_names)
else:
full_name += " " + random.choice(mnames)
if random.random() < 0.1:
full_name += " {} ".format(random.choice(raw_emojis))
else:
full_name += " " + random.choice(lnames)
email = fname.lower() + "@zulip.com"
names.append((full_name, email))
create_users(zulip_realm, names, tos_version=settings.TOS_VERSION)
iago = get_user_by_delivery_email("*****@*****.**", zulip_realm)
do_change_user_role(iago,
UserProfile.ROLE_REALM_ADMINISTRATOR,
acting_user=None)
iago.is_staff = True
iago.save(update_fields=["is_staff"])
desdemona = get_user_by_delivery_email("*****@*****.**",
zulip_realm)
do_change_user_role(desdemona,
UserProfile.ROLE_REALM_OWNER,
acting_user=None)
shiva = get_user_by_delivery_email("*****@*****.**", zulip_realm)
do_change_user_role(shiva,
UserProfile.ROLE_MODERATOR,
acting_user=None)
guest_user = get_user_by_delivery_email("*****@*****.**",
zulip_realm)
guest_user.role = UserProfile.ROLE_GUEST
guest_user.save(update_fields=["role"])
# These bots are directly referenced from code and thus
# are needed for the test suite.
zulip_realm_bots = [
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(
(f"Extra Bot {i}", f"extrabot{i}@zulip.com"))
create_users(zulip_realm,
zulip_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
zoe = get_user_by_delivery_email("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(
zulip_realm,
zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
bot_owner=zoe,
)
aaron = get_user_by_delivery_email("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [
("Outgoing Webhook", "*****@*****.**"),
]
create_users(
zulip_realm,
zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
bot_owner=aaron,
)
outgoing_webhook = get_user("*****@*****.**",
zulip_realm)
add_service(
"outgoing-webhook",
user_profile=outgoing_webhook,
interface=Service.GENERIC,
base_url="http://127.0.0.1:5002",
token=generate_api_key(),
)
# Add the realm internal bots to each realm.
create_if_missing_realm_internal_bots()
# Create public streams.
signups_stream = Realm.INITIAL_PRIVATE_STREAM_NAME
stream_list = [
"Verona",
"Denmark",
"Scotland",
"Venice",
"Rome",
signups_stream,
]
stream_dict: Dict[str, Dict[str, Any]] = {
"Denmark": {
"description": "A Scandinavian country"
},
"Scotland": {
"description": "Located in the United Kingdom"
},
"Venice": {
"description": "A northeastern Italian city"
},
"Rome": {
"description": "Yet another Italian city",
"is_web_public": True
},
}
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams: List[int] = [
Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list
]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users). For the test suite, we have a fixed list of
# subscriptions to make sure test data is consistent
# across platforms.
subscriptions_list: List[Tuple[UserProfile, Recipient]] = []
profiles: Sequence[UserProfile] = (
UserProfile.objects.select_related().filter(
is_bot=False).order_by("email"))
if options["test_suite"]:
subscriptions_map = {
"*****@*****.**": ["Verona"],
"*****@*****.**": ["Verona"],
"*****@*****.**": ["Verona", "Denmark", signups_stream],
"*****@*****.**": [
"Verona",
"Denmark",
"Scotland",
signups_stream,
],
"*****@*****.**": ["Verona", "Denmark", "Scotland"],
"*****@*****.**":
["Verona", "Denmark", "Scotland", "Venice"],
"*****@*****.**":
["Verona", "Denmark", "Scotland", "Venice", "Rome"],
"*****@*****.**": ["Verona"],
"*****@*****.**": [
"Verona",
"Denmark",
"Venice",
signups_stream,
],
"*****@*****.**": ["Verona", "Denmark", "Scotland"],
}
for profile in profiles:
email = profile.delivery_email
if email not in subscriptions_map:
raise Exception(
f"Subscriptions not listed for user {email}")
for stream_name in subscriptions_map[email]:
stream = Stream.objects.get(name=stream_name,
realm=zulip_realm)
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=stream.id)
subscriptions_list.append((profile, r))
else:
num_streams = len(recipient_streams)
num_users = len(profiles)
for i, profile in enumerate(profiles):
# Subscribe to some streams.
fraction = float(i) / num_users
num_recips = int(num_streams * fraction) + 1
for type_id in recipient_streams[:num_recips]:
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=type_id)
subscriptions_list.append((profile, r))
subscriptions_to_add: List[Subscription] = []
event_time = timezone_now()
all_subscription_logs: (List[RealmAuditLog]) = []
i = 0
for profile, recipient in subscriptions_list:
i += 1
color = STREAM_ASSIGNMENT_COLORS[i %
len(STREAM_ASSIGNMENT_COLORS)]
s = Subscription(
recipient=recipient,
user_profile=profile,
is_user_active=profile.is_active,
color=color,
)
subscriptions_to_add.append(s)
log = RealmAuditLog(
realm=profile.realm,
modified_user=profile,
modified_stream_id=recipient.type_id,
event_last_message_id=0,
event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
event_time=event_time,
)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(
zulip_realm,
"Phone number",
CustomProfileField.SHORT_TEXT,
hint="")
biography = try_add_realm_custom_profile_field(
zulip_realm,
"Biography",
CustomProfileField.LONG_TEXT,
hint="What are you known for?",
)
favorite_food = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite food",
CustomProfileField.SHORT_TEXT,
hint="Or drink, if you'd prefer",
)
field_data: ProfileFieldData = {
"vim": {
"text": "Vim",
"order": "1"
},
"emacs": {
"text": "Emacs",
"order": "2"
},
}
favorite_editor = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite editor",
CustomProfileField.SELECT,
field_data=field_data)
birthday = try_add_realm_custom_profile_field(
zulip_realm, "Birthday", CustomProfileField.DATE)
favorite_website = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite website",
CustomProfileField.URL,
hint="Or your personal blog's URL",
)
mentor = try_add_realm_custom_profile_field(
zulip_realm, "Mentor", CustomProfileField.USER)
github_profile = try_add_realm_default_custom_profile_field(
zulip_realm, "github")
# Fill in values for Iago and Hamlet
hamlet = get_user_by_delivery_email("*****@*****.**",
zulip_realm)
do_update_user_custom_profile_data_if_changed(
iago,
[
{
"id": phone_number.id,
"value": "+1-234-567-8901"
},
{
"id": biography.id,
"value": "Betrayer of Othello."
},
{
"id": favorite_food.id,
"value": "Apples"
},
{
"id": favorite_editor.id,
"value": "emacs"
},
{
"id": birthday.id,
"value": "2000-01-01"
},
{
"id": favorite_website.id,
"value": "https://zulip.readthedocs.io/en/latest/"
},
{
"id": mentor.id,
"value": [hamlet.id]
},
{
"id": github_profile.id,
"value": "zulip"
},
],
)
do_update_user_custom_profile_data_if_changed(
hamlet,
[
{
"id": phone_number.id,
"value": "+0-11-23-456-7890"
},
{
"id":
biography.id,
"value":
"I am:\n* The prince of Denmark\n* Nephew to the usurping Claudius",
},
{
"id": favorite_food.id,
"value": "Dark chocolate"
},
{
"id": favorite_editor.id,
"value": "vim"
},
{
"id": birthday.id,
"value": "1900-01-01"
},
{
"id": favorite_website.id,
"value": "https://blog.zulig.org"
},
{
"id": mentor.id,
"value": [iago.id]
},
{
"id": github_profile.id,
"value": "zulipbot"
},
],
)
else:
zulip_realm = get_realm("zulip")
recipient_streams = [
klass.type_id
for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
# Extract a list of all users
user_profiles: List[UserProfile] = list(
UserProfile.objects.filter(is_bot=False))
# Create a test realm emoji.
IMAGE_FILE_PATH = static_path("images/test-images/checkbox.png")
with open(IMAGE_FILE_PATH, "rb") as fp:
check_add_realm_emoji(zulip_realm, "green_tick", iago, fp)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status: int = UserPresence.ACTIVE
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= "H":
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(
user_profile=user,
realm_id=user.realm_id,
client=client,
timestamp=date,
status=status,
)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [
random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])
]
create_alert_words(zulip_realm.id)
# Generate a new set of test data.
create_test_data()
# prepopulate the URL preview/embed data for the links present
# in the config.generate_data.json data set. This makes it
# possible for populate_db to run happily without Internet
# access.
with open("zerver/tests/fixtures/docs_url_preview_data.json",
"rb") as f:
urls_with_preview_data = orjson.loads(f.read())
for url in urls_with_preview_data:
cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)
if options["delete"]:
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)",
"*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm,
testsuite_mit_users,
tos_version=settings.TOS_VERSION)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia, Lear's daughter", "*****@*****.**"),
]
create_users(lear_realm,
testsuite_lear_users,
tos_version=settings.TOS_VERSION)
if not options["test_suite"]:
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
# to imitate emoji insertions in stream names
raw_emojis = ["😎", "😂", "🐱👤"]
zulip_stream_dict: Dict[str, Dict[str, Any]] = {
"devel": {
"description": "For developing"
},
# ビデオゲーム - VideoGames (japanese)
"ビデオゲーム": {
"description":
"Share your favorite video games! {}".format(
raw_emojis[2])
},
"announce": {
"description": "For announcements",
"stream_post_policy": Stream.STREAM_POST_POLICY_ADMINS,
},
"design": {
"description": "For design"
},
"support": {
"description": "For support"
},
"social": {
"description": "For socializing"
},
"test": {
"description": "For testing `code`"
},
"errors": {
"description": "For errors"
},
# 조리법 - Recipes (Korean) , Пельмени - Dumplings (Russian)
"조리법 " + raw_emojis[0]: {
"description":
"Everything cooking, from pasta to Пельмени"
},
}
extra_stream_names = [
"802.11a",
"Ad Hoc Network",
"Augmented Reality",
"Cycling",
"DPI",
"FAQ",
"FiFo",
"commits",
"Control panel",
"desktop",
"компьютеры",
"Data security",
"desktop",
"काम",
"discussions",
"Cloud storage",
"GCI",
"Vaporware",
"Recent Trends",
"issues",
"live",
"Health",
"mobile",
"空間",
"provision",
"hidrógeno",
"HR",
"アニメ",
]
# Add stream names and stream descriptions
for i in range(options["extra_streams"]):
extra_stream_name = random.choice(
extra_stream_names) + " " + str(i)
# to imitate emoji insertions in stream names
if random.random() <= 0.15:
extra_stream_name += random.choice(raw_emojis)
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream(
"announce", zulip_realm)
zulip_realm.save(update_fields=["notifications_stream"])
# Add a few default streams
for default_stream_name in [
"design", "devel", "social", "support"
]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(
default_stream_name,
zulip_realm))
# Now subscribe everyone to these streams
subscribe_users_to_streams(zulip_realm, zulip_stream_dict)
create_user_groups()
if not options["test_suite"]:
# We populate the analytics database here for
# development purpose only
call_command("populate_analytics_db")
threads = options["threads"]
jobs: List[Tuple[int, List[List[int]], Dict[str, Any],
Callable[[str], int], int]] = []
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write,
random.randint(0, 10**10)))
for job in jobs:
generate_and_send_messages(job)
if options["delete"]:
if not options["test_suite"]:
# These bots are not needed by the test suite
# Also, we don't want interacting with each other
# in dev setup.
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm,
internal_zulip_users_nosubs,
bot_type=UserProfile.DEFAULT_BOT)
mark_all_messages_as_read()
self.stdout.write("Successfully populated test database.\n")
def test_add_missing_messages(self) -> None:
recipient_list = [
self.example_user("hamlet"),
self.example_user("iago")
]
for user_profile in recipient_list:
self.subscribe(user_profile, "Denmark")
sender = self.example_user('iago')
realm = sender.realm
sending_client = make_client(name="test suite")
stream_name = 'Denmark'
stream = get_stream(stream_name, realm)
topic_name = 'foo'
def send_fake_message(message_content: str, stream: Stream) -> Message:
recipient = stream.recipient
message = Message(sender=sender,
recipient=recipient,
content=message_content,
date_sent=timezone_now(),
sending_client=sending_client)
message.set_topic_name(topic_name)
message.save()
return message
long_term_idle_user = self.example_user('hamlet')
self.send_stream_message(long_term_idle_user, stream_name)
with self.assertLogs(level='INFO') as info_logs:
do_soft_deactivate_users([long_term_idle_user])
self.assertEqual(info_logs.output, [
'INFO:root:Soft-deactivated batch of 1 users; 0 remain to process'
])
# Test that add_missing_messages() in simplest case of adding a
# message for which UserMessage row doesn't exist for this user.
sent_message = send_fake_message('Test Message 1', stream)
idle_user_msg_list = get_user_messages(long_term_idle_user)
idle_user_msg_count = len(idle_user_msg_list)
self.assertNotEqual(idle_user_msg_list[-1], sent_message)
with queries_captured() as queries:
add_missing_messages(long_term_idle_user)
self.assert_length(queries, 6)
idle_user_msg_list = get_user_messages(long_term_idle_user)
self.assertEqual(len(idle_user_msg_list), idle_user_msg_count + 1)
self.assertEqual(idle_user_msg_list[-1], sent_message)
long_term_idle_user.refresh_from_db()
self.assertEqual(long_term_idle_user.last_active_message_id,
sent_message.id)
# Test that add_missing_messages() only adds messages that aren't
# already present in the UserMessage table. This test works on the
# fact that previous test just above this added a message but didn't
# updated the last_active_message_id field for the user.
sent_message = send_fake_message('Test Message 2', stream)
idle_user_msg_list = get_user_messages(long_term_idle_user)
idle_user_msg_count = len(idle_user_msg_list)
self.assertNotEqual(idle_user_msg_list[-1], sent_message)
with queries_captured() as queries:
add_missing_messages(long_term_idle_user)
self.assert_length(queries, 7)
idle_user_msg_list = get_user_messages(long_term_idle_user)
self.assertEqual(len(idle_user_msg_list), idle_user_msg_count + 1)
self.assertEqual(idle_user_msg_list[-1], sent_message)
long_term_idle_user.refresh_from_db()
self.assertEqual(long_term_idle_user.last_active_message_id,
sent_message.id)
# Test UserMessage rows are created correctly in case of stream
# Subscription was altered by admin while user was away.
# Test for a public stream.
sent_message_list = []
sent_message_list.append(send_fake_message('Test Message 3', stream))
# Alter subscription to stream.
self.unsubscribe(long_term_idle_user, stream_name)
send_fake_message('Test Message 4', stream)
self.subscribe(long_term_idle_user, stream_name)
sent_message_list.append(send_fake_message('Test Message 5', stream))
sent_message_list.reverse()
idle_user_msg_list = get_user_messages(long_term_idle_user)
idle_user_msg_count = len(idle_user_msg_list)
for sent_message in sent_message_list:
self.assertNotEqual(idle_user_msg_list.pop(), sent_message)
with queries_captured() as queries:
add_missing_messages(long_term_idle_user)
self.assert_length(queries, 6)
idle_user_msg_list = get_user_messages(long_term_idle_user)
self.assertEqual(len(idle_user_msg_list), idle_user_msg_count + 2)
for sent_message in sent_message_list:
self.assertEqual(idle_user_msg_list.pop(), sent_message)
long_term_idle_user.refresh_from_db()
self.assertEqual(long_term_idle_user.last_active_message_id,
sent_message_list[0].id)
# Test consecutive subscribe/unsubscribe in a public stream
sent_message_list = []
sent_message_list.append(send_fake_message('Test Message 6', stream))
# Unsubscribe from stream and then immediately subscribe back again.
self.unsubscribe(long_term_idle_user, stream_name)
self.subscribe(long_term_idle_user, stream_name)
sent_message_list.append(send_fake_message('Test Message 7', stream))
# Again unsubscribe from stream and send a message.
# This will make sure that if initially in a unsubscribed state
# a consecutive subscribe/unsubscribe doesn't misbehave.
self.unsubscribe(long_term_idle_user, stream_name)
send_fake_message('Test Message 8', stream)
# Do a subscribe and unsubscribe immediately.
self.subscribe(long_term_idle_user, stream_name)
self.unsubscribe(long_term_idle_user, stream_name)
sent_message_list.reverse()
idle_user_msg_list = get_user_messages(long_term_idle_user)
idle_user_msg_count = len(idle_user_msg_list)
for sent_message in sent_message_list:
self.assertNotEqual(idle_user_msg_list.pop(), sent_message)
with queries_captured() as queries:
add_missing_messages(long_term_idle_user)
self.assert_length(queries, 6)
idle_user_msg_list = get_user_messages(long_term_idle_user)
self.assertEqual(len(idle_user_msg_list), idle_user_msg_count + 2)
for sent_message in sent_message_list:
self.assertEqual(idle_user_msg_list.pop(), sent_message)
long_term_idle_user.refresh_from_db()
self.assertEqual(long_term_idle_user.last_active_message_id,
sent_message_list[0].id)
# Test for when user unsubscribes before soft deactivation
# (must reactivate them in order to do this).
do_soft_activate_users([long_term_idle_user])
self.subscribe(long_term_idle_user, stream_name)
# Send a real message to update last_active_message_id
sent_message_id = self.send_stream_message(sender, stream_name,
'Test Message 9')
self.unsubscribe(long_term_idle_user, stream_name)
# Soft deactivate and send another message to the unsubscribed stream.
with self.assertLogs(level='INFO') as info_logs:
do_soft_deactivate_users([long_term_idle_user])
self.assertEqual(info_logs.output, [
'INFO:root:Soft-deactivated batch of 1 users; 0 remain to process'
])
send_fake_message('Test Message 10', stream)
idle_user_msg_list = get_user_messages(long_term_idle_user)
idle_user_msg_count = len(idle_user_msg_list)
self.assertEqual(idle_user_msg_list[-1].id, sent_message_id)
with queries_captured() as queries:
add_missing_messages(long_term_idle_user)
# There are no streams to fetch missing messages from, so
# the Message.objects query will be avoided.
self.assert_length(queries, 4)
idle_user_msg_list = get_user_messages(long_term_idle_user)
# No new UserMessage rows should have been created.
self.assertEqual(len(idle_user_msg_list), idle_user_msg_count)
# Note: At this point in this test we have long_term_idle_user
# unsubscribed from the 'Denmark' stream.
# Test for a Private Stream.
stream_name = "Core"
private_stream = self.make_stream('Core', invite_only=True)
self.subscribe(self.example_user("iago"), stream_name)
sent_message_list = []
send_fake_message('Test Message 11', private_stream)
self.subscribe(self.example_user("hamlet"), stream_name)
sent_message_list.append(
send_fake_message('Test Message 12', private_stream))
self.unsubscribe(long_term_idle_user, stream_name)
send_fake_message('Test Message 13', private_stream)
self.subscribe(long_term_idle_user, stream_name)
sent_message_list.append(
send_fake_message('Test Message 14', private_stream))
sent_message_list.reverse()
idle_user_msg_list = get_user_messages(long_term_idle_user)
idle_user_msg_count = len(idle_user_msg_list)
for sent_message in sent_message_list:
self.assertNotEqual(idle_user_msg_list.pop(), sent_message)
with queries_captured() as queries:
add_missing_messages(long_term_idle_user)
self.assert_length(queries, 6)
idle_user_msg_list = get_user_messages(long_term_idle_user)
self.assertEqual(len(idle_user_msg_list), idle_user_msg_count + 2)
for sent_message in sent_message_list:
self.assertEqual(idle_user_msg_list.pop(), sent_message)
long_term_idle_user.refresh_from_db()
self.assertEqual(long_term_idle_user.last_active_message_id,
sent_message_list[0].id)
def test_user_message_filter(self) -> None:
# In this test we are basically testing out the logic used out in
# do_send_messages() in action.py for filtering the messages for which
# UserMessage rows should be created for a soft-deactivated user.
AlertWord.objects.all().delete()
long_term_idle_user = self.example_user("hamlet")
cordelia = self.example_user("cordelia")
sender = self.example_user("iago")
stream_name = "Brand New Stream"
topic_name = "foo"
realm_id = cordelia.realm_id
self.subscribe(long_term_idle_user, stream_name)
self.subscribe(cordelia, stream_name)
self.subscribe(sender, stream_name)
stream_id = get_stream(stream_name, cordelia.realm).id
def send_stream_message(content: str) -> None:
self.send_stream_message(sender, stream_name, content, topic_name)
def send_personal_message(content: str) -> None:
self.send_personal_message(sender, self.example_user("hamlet"),
content)
self.send_stream_message(long_term_idle_user, stream_name)
with self.assertLogs(logger_string, level="INFO") as info_logs:
do_soft_deactivate_users([long_term_idle_user])
self.assertEqual(
info_logs.output,
[
f"INFO:{logger_string}:Soft deactivated user {long_term_idle_user.id}",
f"INFO:{logger_string}:Soft-deactivated batch of 1 users; 0 remain to process",
],
)
def assert_um_count(user: UserProfile, count: int) -> None:
user_messages = get_user_messages(user)
self.assert_length(user_messages, count)
def assert_last_um_content(user: UserProfile,
content: str,
negate: bool = False) -> None:
user_messages = get_user_messages(user)
if negate:
self.assertNotEqual(user_messages[-1].content, content)
else:
self.assertEqual(user_messages[-1].content, content)
def assert_num_possible_users(
expected_count: int,
*,
possible_wildcard_mention: bool = False,
possibly_mentioned_user_ids: AbstractSet[int] = set(),
) -> None:
self.assertEqual(
len(
get_subscriptions_for_send_message(
realm_id=realm_id,
stream_id=stream_id,
possible_wildcard_mention=possible_wildcard_mention,
possibly_mentioned_user_ids=possibly_mentioned_user_ids,
)),
expected_count,
)
def assert_stream_message_sent_to_idle_user(
content: str,
*,
possible_wildcard_mention: bool = False,
possibly_mentioned_user_ids: AbstractSet[int] = set(),
) -> None:
assert_num_possible_users(
expected_count=3,
possible_wildcard_mention=possible_wildcard_mention,
possibly_mentioned_user_ids=possibly_mentioned_user_ids,
)
general_user_msg_count = len(get_user_messages(cordelia))
soft_deactivated_user_msg_count = len(
get_user_messages(long_term_idle_user))
send_stream_message(content)
assert_um_count(long_term_idle_user,
soft_deactivated_user_msg_count + 1)
assert_um_count(cordelia, general_user_msg_count + 1)
assert_last_um_content(long_term_idle_user, content)
assert_last_um_content(cordelia, content)
def assert_stream_message_not_sent_to_idle_user(
content: str,
*,
possibly_mentioned_user_ids: AbstractSet[int] = set(),
false_alarm_row: bool = False,
) -> None:
if false_alarm_row:
# We will query for our idle user if he has **ANY** alert
# words, but we won't actually write a UserMessage row until
# we truly parse the message. We also get false alarms for
# messages with quoted mentions.
assert_num_possible_users(
3, possibly_mentioned_user_ids=possibly_mentioned_user_ids)
else:
assert_num_possible_users(2)
general_user_msg_count = len(get_user_messages(cordelia))
soft_deactivated_user_msg_count = len(
get_user_messages(long_term_idle_user))
send_stream_message(content)
assert_um_count(long_term_idle_user,
soft_deactivated_user_msg_count)
assert_um_count(cordelia, general_user_msg_count + 1)
assert_last_um_content(long_term_idle_user, content, negate=True)
assert_last_um_content(cordelia, content)
# Test that sending a message to a stream with soft deactivated user
# doesn't end up creating UserMessage row for deactivated user.
assert_stream_message_not_sent_to_idle_user("Test message 1")
sub = get_subscription(stream_name, long_term_idle_user)
# Sub settings override user settings.
sub.push_notifications = True
sub.save()
assert_stream_message_sent_to_idle_user("Sub push")
sub.push_notifications = False
sub.save()
assert_stream_message_not_sent_to_idle_user("Sub no push")
# Let user defaults take over
sub.push_notifications = None
sub.save()
long_term_idle_user.enable_stream_push_notifications = True
long_term_idle_user.save()
assert_stream_message_sent_to_idle_user("User push")
long_term_idle_user.enable_stream_push_notifications = False
long_term_idle_user.save()
assert_stream_message_not_sent_to_idle_user("User no push")
# Sub settings override user settings.
sub.email_notifications = True
sub.save()
assert_stream_message_sent_to_idle_user("Sub email")
sub.email_notifications = False
sub.save()
assert_stream_message_not_sent_to_idle_user("Sub no email")
# Let user defaults take over
sub.email_notifications = None
sub.save()
long_term_idle_user.enable_stream_email_notifications = True
long_term_idle_user.save()
assert_stream_message_sent_to_idle_user("User email")
long_term_idle_user.enable_stream_email_notifications = False
long_term_idle_user.save()
assert_stream_message_not_sent_to_idle_user("User no email")
# Test sending a private message to soft deactivated user creates
# UserMessage row.
soft_deactivated_user_msg_count = len(
get_user_messages(long_term_idle_user))
message = "Test PM"
send_personal_message(message)
assert_um_count(long_term_idle_user,
soft_deactivated_user_msg_count + 1)
assert_last_um_content(long_term_idle_user, message)
# Test UserMessage row is created while user is deactivated if
# user itself is mentioned.
assert_stream_message_sent_to_idle_user(
"Test @**King Hamlet** mention",
possibly_mentioned_user_ids={long_term_idle_user.id},
)
assert_stream_message_not_sent_to_idle_user(
"Test `@**King Hamlet**` mention",
possibly_mentioned_user_ids={long_term_idle_user.id},
false_alarm_row=True,
)
# Test UserMessage row is not created while user is deactivated if
# anyone is mentioned but the user.
assert_stream_message_not_sent_to_idle_user(
"Test @**Cordelia, Lear's daughter** mention")
# Test UserMessage row is created while user is deactivated if
# there is a wildcard mention such as @all or @everyone
assert_stream_message_sent_to_idle_user("Test @**all** mention",
possible_wildcard_mention=True)
assert_stream_message_sent_to_idle_user("Test @**everyone** mention",
possible_wildcard_mention=True)
assert_stream_message_sent_to_idle_user("Test @**stream** mention",
possible_wildcard_mention=True)
assert_stream_message_not_sent_to_idle_user("Test @**bogus** mention")
# Test UserMessage row is created while user is deactivated if there
# is a alert word in message.
do_add_alert_words(long_term_idle_user, ["test_alert_word"])
assert_stream_message_sent_to_idle_user("Testing test_alert_word")
do_add_alert_words(cordelia, ["cordelia"])
assert_stream_message_not_sent_to_idle_user("cordelia",
false_alarm_row=True)
# Test UserMessage row is not created while user is deactivated if
# message is a me message.
assert_stream_message_not_sent_to_idle_user("/me says test",
false_alarm_row=True)
# Sanity check after removing the alert word for Hamlet.
AlertWord.objects.filter(user_profile=long_term_idle_user).delete()
assert_stream_message_not_sent_to_idle_user("no alert words")
def unsubscribe(self, user_profile, stream_name):
# type: (UserProfile, Text) -> None
stream = get_stream(stream_name, user_profile.realm)
bulk_remove_subscriptions([user_profile], [stream])
def get_subscription(stream_name, user_profile):
# type: (Text, UserProfile) -> Subscription
stream = get_stream(stream_name, user_profile.realm)
recipient = get_recipient(Recipient.STREAM, stream.id)
return Subscription.objects.get(user_profile=user_profile,
recipient=recipient, active=True)
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write(
"Error! More than 100% of messages allocated.\n")
return
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our two default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
zulip_realm = Realm.objects.create(
string_id="zulip",
name="Zulip Dev",
restricted_to_domain=True,
description=
"The Zulip development environment default organization."
" It's great for testing!",
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
if options["test_suite"]:
mit_realm = Realm.objects.create(string_id="zephyr",
name="MIT",
restricted_to_domain=True,
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = Realm.objects.create(string_id="lear",
name="Lear & Co.",
restricted_to_domain=False,
invite_required=False,
org_type=Realm.CORPORATE)
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
("Polonius", "*****@*****.**"),
]
for i in range(options["extra_users"]):
names.append(
('Extra User %d' % (i, ), '*****@*****.**' % (i, )))
create_users(zulip_realm, names)
iago = get_user("*****@*****.**", zulip_realm)
do_change_is_admin(iago, True)
iago.is_staff = True
iago.save(update_fields=['is_staff'])
guest_user = get_user("*****@*****.**", zulip_realm)
guest_user.is_guest = True
guest_user.save(update_fields=['is_guest'])
# These bots are directly referenced from code and thus
# are needed for the test suite.
all_realm_bots = [
(bot['name'],
bot['email_template'] % (settings.INTERNAL_BOT_DOMAIN, ))
for bot in settings.INTERNAL_BOTS
]
zulip_realm_bots = [
("Zulip New User Bot", "*****@*****.**"),
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
("Welcome Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(
('Extra Bot %d' % (i, ), '*****@*****.**' % (i, )))
zulip_realm_bots.extend(all_realm_bots)
create_users(zulip_realm,
zulip_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
# Initialize the email gateway bot as an API Super User
email_gateway_bot = get_system_bot(settings.EMAIL_GATEWAY_BOT)
email_gateway_bot.is_api_super_user = True
email_gateway_bot.save()
zoe = get_user("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(zulip_realm,
zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
bot_owner=zoe)
aaron = get_user("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [("Outgoing Webhook",
"*****@*****.**")]
create_users(zulip_realm,
zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
bot_owner=aaron)
# TODO: Clean up this initial bot creation code
Service.objects.create(
name="test",
user_profile=get_user("*****@*****.**",
zulip_realm),
base_url="http://127.0.0.1:5002/bots/followup",
token="abcd1234",
interface=1)
# Create public streams.
stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
stream_dict = {
"Verona": {
"description": "A city in Italy",
"invite_only": False,
"is_web_public": False
},
"Denmark": {
"description": "A Scandinavian country",
"invite_only": False,
"is_web_public": False
},
"Scotland": {
"description": "Located in the United Kingdom",
"invite_only": False,
"is_web_public": False
},
"Venice": {
"description": "A northeastern Italian city",
"invite_only": False,
"is_web_public": False
},
"Rome": {
"description": "Yet another Italian city",
"invite_only": False,
"is_web_public": True
}
} # type: Dict[str, Dict[str, Any]]
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams = [
Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list
] # type: List[int]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users).
subscriptions_to_add = [] # type: List[Subscription]
event_time = timezone_now()
all_subscription_logs = [] # type: (List[RealmAuditLog])
profiles = UserProfile.objects.select_related().filter(
is_bot=False, is_guest=False).order_by(
"email") # type: Sequence[UserProfile]
for i, profile in enumerate(profiles):
# Subscribe to some streams.
for type_id in recipient_streams[:int(
len(recipient_streams) * float(i) / len(profiles)) +
1]:
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=type_id)
s = Subscription(recipient=r,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[
i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream_id=type_id,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(
zulip_realm,
"Phone number",
CustomProfileField.SHORT_TEXT,
hint='')
biography = try_add_realm_custom_profile_field(
zulip_realm,
"Biography",
CustomProfileField.LONG_TEXT,
hint='What are you known for?')
favorite_food = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite food",
CustomProfileField.SHORT_TEXT,
hint="Or drink, if you'd prefer")
field_data = {
'vim': {
'text': 'Vim',
'order': '1'
},
'emacs': {
'text': 'Emacs',
'order': '2'
},
}
favorite_editor = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite editor",
CustomProfileField.CHOICE,
field_data=field_data)
birthday = try_add_realm_custom_profile_field(
zulip_realm, "Birthday", CustomProfileField.DATE)
favorite_website = try_add_realm_custom_profile_field(
zulip_realm,
"GitHub profile",
CustomProfileField.URL,
hint="Or your personal blog's URL")
# Fill in values for Iago and Hamlet
hamlet = get_user("*****@*****.**", zulip_realm)
do_update_user_custom_profile_data(iago, [
{
"id": phone_number.id,
"value": "+1-234-567-8901"
},
{
"id": biography.id,
"value": "Betrayer of Othello."
},
{
"id": favorite_food.id,
"value": "Apples"
},
{
"id": favorite_editor.id,
"value": "emacs"
},
{
"id": birthday.id,
"value": "2000-1-1"
},
{
"id": favorite_website.id,
"value": "https://github.com/zulip/zulip"
},
])
do_update_user_custom_profile_data(hamlet, [
{
"id": phone_number.id,
"value": "+0-11-23-456-7890"
},
{
"id": biography.id,
"value": "Prince of Denmark, and other things!"
},
{
"id": favorite_food.id,
"value": "Dark chocolate"
},
{
"id": favorite_editor.id,
"value": "vim"
},
{
"id": birthday.id,
"value": "1900-1-1"
},
{
"id": favorite_website.id,
"value": "https://blog.zulig.org"
},
])
else:
zulip_realm = get_realm("zulip")
recipient_streams = [
klass.type_id
for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
# Extract a list of all users
user_profiles = list(UserProfile.objects.filter(
is_bot=False)) # type: List[UserProfile]
# Create a test realm emoji.
IMAGE_FILE_PATH = os.path.join(settings.STATIC_ROOT, 'images',
'test-images', 'checkbox.png')
with open(IMAGE_FILE_PATH, 'rb') as fp:
check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status = UserPresence.ACTIVE # type: int
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= 'H':
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(user_profile=user,
client=client,
timestamp=date,
status=status)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [
random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])
]
# Generate a new set of test data.
create_test_data()
# prepopulate the URL preview/embed data for the links present
# in the config.generate_data.json data set. This makes it
# possible for populate_db to run happily without Internet
# access.
with open("zerver/tests/fixtures/docs_url_preview_data.json",
"r") as f:
urls_with_preview_data = ujson.load(f)
for url in urls_with_preview_data:
cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)
threads = options["threads"]
jobs = [
] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write,
random.randint(0, 10**10)))
for job in jobs:
send_messages(job)
if options["delete"]:
# Create the "website" and "API" clients; if we don't, the
# default values in zerver/decorators.py will not work
# with the Django test suite.
get_client("website")
get_client("API")
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)",
"*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm, testsuite_mit_users)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
]
create_users(lear_realm, testsuite_lear_users)
if not options["test_suite"]:
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
zulip_stream_dict = {
"devel": {
"description": "For developing",
"invite_only": False,
"is_web_public": False
},
"all": {
"description": "For everything",
"invite_only": False,
"is_web_public": False
},
"announce": {
"description": "For announcements",
"invite_only": False,
"is_web_public": False
},
"design": {
"description": "For design",
"invite_only": False,
"is_web_public": False
},
"support": {
"description": "For support",
"invite_only": False,
"is_web_public": False
},
"social": {
"description": "For socializing",
"invite_only": False,
"is_web_public": False
},
"test": {
"description": "For testing",
"invite_only": False,
"is_web_public": False
},
"errors": {
"description": "For errors",
"invite_only": False,
"is_web_public": False
},
"sales": {
"description": "For sales discussion",
"invite_only": False,
"is_web_public": False
}
} # type: Dict[str, Dict[str, Any]]
# Calculate the maximum number of digits in any extra stream's
# number, since a stream with name "Extra Stream 3" could show
# up after "Extra Stream 29". (Used later to pad numbers with
# 0s).
maximum_digits = len(str(options['extra_streams'] - 1))
for i in range(options['extra_streams']):
# Pad the number with 0s based on `maximum_digits`.
number_str = str(i).zfill(maximum_digits)
extra_stream_name = 'Extra Stream ' + number_str
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
"invite_only": False,
"is_web_public": False,
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream(
"announce", zulip_realm)
zulip_realm.save(update_fields=['notifications_stream'])
# Add a few default streams
for default_stream_name in [
"design", "devel", "social", "support"
]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(
default_stream_name,
zulip_realm))
# Now subscribe everyone to these streams
subscriptions_to_add = []
event_time = timezone_now()
all_subscription_logs = []
profiles = UserProfile.objects.select_related().filter(
realm=zulip_realm)
for i, stream_name in enumerate(zulip_stream_dict):
stream = Stream.objects.get(name=stream_name,
realm=zulip_realm)
recipient = Recipient.objects.get(type=Recipient.STREAM,
type_id=stream.id)
for profile in profiles:
# Subscribe to some streams.
s = Subscription(
recipient=recipient,
user_profile=profile,
color=STREAM_ASSIGNMENT_COLORS[
i % len(STREAM_ASSIGNMENT_COLORS)])
subscriptions_to_add.append(s)
log = RealmAuditLog(realm=profile.realm,
modified_user=profile,
modified_stream=stream,
event_last_message_id=0,
event_type='subscription_created',
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# These bots are not needed by the test suite
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm,
internal_zulip_users_nosubs,
bot_type=UserProfile.DEFAULT_BOT)
zulip_cross_realm_bots = [
("Zulip Feedback Bot", "*****@*****.**"),
]
create_users(zulip_realm,
zulip_cross_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
# Mark all messages as read
UserMessage.objects.all().update(flags=UserMessage.flags.read)
if not options["test_suite"]:
# Update pointer of each user to point to the last message in their
# UserMessage rows with sender_id=user_profile_id.
users = list(
UserMessage.objects.filter(message__sender_id=F(
'user_profile_id')).values('user_profile_id').annotate(
pointer=Max('message_id')))
for user in users:
UserProfile.objects.filter(
id=user['user_profile_id']).update(
pointer=user['pointer'])
create_user_groups()
self.stdout.write("Successfully populated test database.\n")
def test_topic_delete(self) -> None:
initial_last_msg_id = self.get_last_message().id
stream_name = "new_stream"
topic_name = "new topic 2"
# NON-ADMIN USER
user_profile = self.example_user("hamlet")
self.subscribe(user_profile, stream_name)
# Send message
stream = get_stream(stream_name, user_profile.realm)
self.send_stream_message(user_profile,
stream_name,
topic_name=topic_name)
last_msg_id = self.send_stream_message(user_profile,
stream_name,
topic_name=topic_name)
# Deleting the topic
self.login_user(user_profile)
endpoint = "/json/streams/" + str(stream.id) + "/delete_topic"
result = self.client_post(
endpoint,
{
"topic_name": topic_name,
},
)
self.assert_json_error(result, "Must be an organization administrator")
self.assertTrue(Message.objects.filter(id=last_msg_id).exists())
# Make stream private with limited history
do_change_stream_permission(stream,
invite_only=True,
history_public_to_subscribers=False,
acting_user=user_profile)
# ADMIN USER subscribed now
user_profile = self.example_user("iago")
self.subscribe(user_profile, stream_name)
self.login_user(user_profile)
new_last_msg_id = self.send_stream_message(user_profile,
stream_name,
topic_name=topic_name)
# Now admin deletes all messages in topic -- which should only
# delete new_last_msg_id, i.e. the one sent since they joined.
self.assertEqual(self.get_last_message().id, new_last_msg_id)
result = self.client_post(
endpoint,
{
"topic_name": topic_name,
},
)
self.assert_json_success(result)
self.assertTrue(Message.objects.filter(id=last_msg_id).exists())
# Try to delete all messages in the topic again. There are no messages accessible
# to the administrator, so this should do nothing.
result = self.client_post(
endpoint,
{
"topic_name": topic_name,
},
)
self.assert_json_success(result)
self.assertTrue(Message.objects.filter(id=last_msg_id).exists())
# Make the stream's history public to subscribers
do_change_stream_permission(stream,
invite_only=True,
history_public_to_subscribers=True,
acting_user=user_profile)
# Delete the topic should now remove all messages
result = self.client_post(
endpoint,
{
"topic_name": topic_name,
},
)
self.assert_json_success(result)
self.assertFalse(Message.objects.filter(id=last_msg_id).exists())
self.assertTrue(
Message.objects.filter(id=initial_last_msg_id).exists())
# Delete again, to test the edge case of deleting an empty topic.
result = self.client_post(
endpoint,
{
"topic_name": topic_name,
},
)
self.assert_json_success(result)
self.assertFalse(Message.objects.filter(id=last_msg_id).exists())
self.assertTrue(
Message.objects.filter(id=initial_last_msg_id).exists())
def handle(self, **options: Any) -> None:
if options["percent_huddles"] + options["percent_personals"] > 100:
self.stderr.write(
"Error! More than 100% of messages allocated.\n")
return
# Get consistent data for backend tests.
if options["test_suite"]:
random.seed(0)
if options["delete"]:
# Start by clearing all the data in our database
clear_database()
# Create our three default realms
# Could in theory be done via zerver.lib.actions.do_create_realm, but
# welcome-bot (needed for do_create_realm) hasn't been created yet
create_internal_realm()
zulip_realm = Realm.objects.create(
string_id="zulip",
name="Zulip Dev",
emails_restricted_to_domains=True,
description=
"The Zulip development environment default organization."
" It's great for testing!",
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=zulip_realm, domain="zulip.com")
if options["test_suite"]:
mit_realm = Realm.objects.create(
string_id="zephyr",
name="MIT",
emails_restricted_to_domains=True,
invite_required=False,
org_type=Realm.CORPORATE)
RealmDomain.objects.create(realm=mit_realm, domain="mit.edu")
lear_realm = Realm.objects.create(
string_id="lear",
name="Lear & Co.",
emails_restricted_to_domains=False,
invite_required=False,
org_type=Realm.CORPORATE)
# Create test Users (UserProfiles are automatically created,
# as are subscriptions to the ability to receive personals).
names = [
("Zoe", "*****@*****.**"),
("Othello, the Moor of Venice", "*****@*****.**"),
("Iago", "*****@*****.**"),
("Prospero from The Tempest", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
("King Hamlet", "*****@*****.**"),
("aaron", "*****@*****.**"),
("Polonius", "*****@*****.**"),
]
# For testing really large batches:
# Create extra users with semi realistic names to make search
# functions somewhat realistic. We'll still create 1000 users
# like Extra222 User for some predicability.
num_names = options['extra_users']
num_boring_names = 1000
for i in range(min(num_names, num_boring_names)):
full_name = 'Extra%03d User' % (i, )
names.append((full_name, '*****@*****.**' % (i, )))
if num_names > num_boring_names:
fnames = [
'Amber', 'Arpita', 'Bob', 'Cindy', 'Daniela', 'Dan',
'Dinesh', 'Faye', 'François', 'George', 'Hank', 'Irene',
'James', 'Janice', 'Jenny', 'Jill', 'John', 'Kate',
'Katelyn', 'Kobe', 'Lexi', 'Manish', 'Mark', 'Matt',
'Mayna', 'Michael', 'Pete', 'Peter', 'Phil', 'Phillipa',
'Preston', 'Sally', 'Scott', 'Sandra', 'Steve',
'Stephanie', 'Vera'
]
mnames = ['de', 'van', 'von', 'Shaw', 'T.']
lnames = [
'Adams', 'Agarwal', 'Beal', 'Benson', 'Bonita', 'Davis',
'George', 'Harden', 'James', 'Jones', 'Johnson', 'Jordan',
'Lee', 'Leonard', 'Singh', 'Smith', 'Patel', 'Towns',
'Wall'
]
for i in range(num_boring_names, num_names):
fname = random.choice(fnames) + str(i)
full_name = fname
if random.random() < 0.7:
if random.random() < 0.5:
full_name += ' ' + random.choice(mnames)
full_name += ' ' + random.choice(lnames)
email = fname.lower() + '@zulip.com'
names.append((full_name, email))
create_users(zulip_realm, names, tos_version=settings.TOS_VERSION)
iago = get_user("*****@*****.**", zulip_realm)
do_change_is_admin(iago, True)
iago.is_staff = True
iago.save(update_fields=['is_staff'])
guest_user = get_user("*****@*****.**", zulip_realm)
guest_user.role = UserProfile.ROLE_GUEST
guest_user.save(update_fields=['role'])
# These bots are directly referenced from code and thus
# are needed for the test suite.
zulip_realm_bots = [
("Zulip Error Bot", "*****@*****.**"),
("Zulip Default Bot", "*****@*****.**"),
]
for i in range(options["extra_bots"]):
zulip_realm_bots.append(
('Extra Bot %d' % (i, ), '*****@*****.**' % (i, )))
create_users(zulip_realm,
zulip_realm_bots,
bot_type=UserProfile.DEFAULT_BOT)
zoe = get_user("*****@*****.**", zulip_realm)
zulip_webhook_bots = [
("Zulip Webhook Bot", "*****@*****.**"),
]
# If a stream is not supplied in the webhook URL, the webhook
# will (in some cases) send the notification as a PM to the
# owner of the webhook bot, so bot_owner can't be None
create_users(zulip_realm,
zulip_webhook_bots,
bot_type=UserProfile.INCOMING_WEBHOOK_BOT,
bot_owner=zoe)
aaron = get_user("*****@*****.**", zulip_realm)
zulip_outgoing_bots = [("Outgoing Webhook",
"*****@*****.**")]
create_users(zulip_realm,
zulip_outgoing_bots,
bot_type=UserProfile.OUTGOING_WEBHOOK_BOT,
bot_owner=aaron)
outgoing_webhook = get_user("*****@*****.**",
zulip_realm)
add_service("outgoing-webhook",
user_profile=outgoing_webhook,
interface=Service.GENERIC,
base_url="http://127.0.0.1:5002",
token=generate_api_key())
# Add the realm internl bots to each realm.
create_if_missing_realm_internal_bots()
# Create public streams.
stream_list = ["Verona", "Denmark", "Scotland", "Venice", "Rome"]
stream_dict = {
"Verona": {
"description": "A city in Italy"
},
"Denmark": {
"description": "A Scandinavian country"
},
"Scotland": {
"description": "Located in the United Kingdom"
},
"Venice": {
"description": "A northeastern Italian city"
},
"Rome": {
"description": "Yet another Italian city",
"is_web_public": True
}
} # type: Dict[str, Dict[str, Any]]
bulk_create_streams(zulip_realm, stream_dict)
recipient_streams = [
Stream.objects.get(name=name, realm=zulip_realm).id
for name in stream_list
] # type: List[int]
# Create subscriptions to streams. The following
# algorithm will give each of the users a different but
# deterministic subset of the streams (given a fixed list
# of users). For the test suite, we have a fixed list of
# subscriptions to make sure test data is consistent
# across platforms.
subscriptions_list = [
] # type: List[Tuple[UserProfile, Recipient]]
profiles = UserProfile.objects.select_related().filter(
is_bot=False).order_by("email") # type: Sequence[UserProfile]
if options["test_suite"]:
subscriptions_map = {
'*****@*****.**': ['Verona'],
'*****@*****.**': ['Verona'],
'*****@*****.**': ['Verona', 'Denmark'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
'*****@*****.**': ['Verona', 'Denmark', 'Scotland'],
'*****@*****.**':
['Verona', 'Denmark', 'Scotland', 'Venice'],
'*****@*****.**':
['Verona', 'Denmark', 'Scotland', 'Venice', 'Rome'],
'*****@*****.**': ['Verona'],
}
for profile in profiles:
if profile.email not in subscriptions_map:
raise Exception(
'Subscriptions not listed for user %s' %
(profile.email, ))
for stream_name in subscriptions_map[profile.email]:
stream = Stream.objects.get(name=stream_name)
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=stream.id)
subscriptions_list.append((profile, r))
else:
num_streams = len(recipient_streams)
num_users = len(profiles)
for i, profile in enumerate(profiles):
# Subscribe to some streams.
fraction = float(i) / num_users
num_recips = int(num_streams * fraction) + 1
for type_id in recipient_streams[:num_recips]:
r = Recipient.objects.get(type=Recipient.STREAM,
type_id=type_id)
subscriptions_list.append((profile, r))
subscriptions_to_add = [] # type: List[Subscription]
event_time = timezone_now()
all_subscription_logs = [] # type: (List[RealmAuditLog])
i = 0
for profile, recipient in subscriptions_list:
i += 1
color = STREAM_ASSIGNMENT_COLORS[i %
len(STREAM_ASSIGNMENT_COLORS)]
s = Subscription(recipient=recipient,
user_profile=profile,
color=color)
subscriptions_to_add.append(s)
log = RealmAuditLog(
realm=profile.realm,
modified_user=profile,
modified_stream_id=recipient.type_id,
event_last_message_id=0,
event_type=RealmAuditLog.SUBSCRIPTION_CREATED,
event_time=event_time)
all_subscription_logs.append(log)
Subscription.objects.bulk_create(subscriptions_to_add)
RealmAuditLog.objects.bulk_create(all_subscription_logs)
# Create custom profile field data
phone_number = try_add_realm_custom_profile_field(
zulip_realm,
"Phone number",
CustomProfileField.SHORT_TEXT,
hint='')
biography = try_add_realm_custom_profile_field(
zulip_realm,
"Biography",
CustomProfileField.LONG_TEXT,
hint='What are you known for?')
favorite_food = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite food",
CustomProfileField.SHORT_TEXT,
hint="Or drink, if you'd prefer")
field_data = {
'vim': {
'text': 'Vim',
'order': '1'
},
'emacs': {
'text': 'Emacs',
'order': '2'
},
} # type: ProfileFieldData
favorite_editor = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite editor",
CustomProfileField.CHOICE,
field_data=field_data)
birthday = try_add_realm_custom_profile_field(
zulip_realm, "Birthday", CustomProfileField.DATE)
favorite_website = try_add_realm_custom_profile_field(
zulip_realm,
"Favorite website",
CustomProfileField.URL,
hint="Or your personal blog's URL")
mentor = try_add_realm_custom_profile_field(
zulip_realm, "Mentor", CustomProfileField.USER)
github_profile = try_add_realm_default_custom_profile_field(
zulip_realm, "github")
# Fill in values for Iago and Hamlet
hamlet = get_user("*****@*****.**", zulip_realm)
do_update_user_custom_profile_data_if_changed(
iago, [
{
"id": phone_number.id,
"value": "+1-234-567-8901"
},
{
"id": biography.id,
"value": "Betrayer of Othello."
},
{
"id": favorite_food.id,
"value": "Apples"
},
{
"id": favorite_editor.id,
"value": "emacs"
},
{
"id": birthday.id,
"value": "2000-1-1"
},
{
"id": favorite_website.id,
"value": "https://zulip.readthedocs.io/en/latest/"
},
{
"id": mentor.id,
"value": [hamlet.id]
},
{
"id": github_profile.id,
"value": 'zulip'
},
])
do_update_user_custom_profile_data_if_changed(
hamlet, [
{
"id": phone_number.id,
"value": "+0-11-23-456-7890"
},
{
"id":
biography.id,
"value":
"I am:\n* The prince of Denmark\n* Nephew to the usurping Claudius",
},
{
"id": favorite_food.id,
"value": "Dark chocolate"
},
{
"id": favorite_editor.id,
"value": "vim"
},
{
"id": birthday.id,
"value": "1900-1-1"
},
{
"id": favorite_website.id,
"value": "https://blog.zulig.org"
},
{
"id": mentor.id,
"value": [iago.id]
},
{
"id": github_profile.id,
"value": 'zulipbot'
},
])
else:
zulip_realm = get_realm("zulip")
recipient_streams = [
klass.type_id
for klass in Recipient.objects.filter(type=Recipient.STREAM)
]
# Extract a list of all users
user_profiles = list(UserProfile.objects.filter(
is_bot=False)) # type: List[UserProfile]
# Create a test realm emoji.
IMAGE_FILE_PATH = static_path('images/test-images/checkbox.png')
with open(IMAGE_FILE_PATH, 'rb') as fp:
check_add_realm_emoji(zulip_realm, 'green_tick', iago, fp)
if not options["test_suite"]:
# Populate users with some bar data
for user in user_profiles:
status = UserPresence.ACTIVE # type: int
date = timezone_now()
client = get_client("website")
if user.full_name[0] <= 'H':
client = get_client("ZulipAndroid")
UserPresence.objects.get_or_create(user_profile=user,
client=client,
timestamp=date,
status=status)
user_profiles_ids = [user_profile.id for user_profile in user_profiles]
# Create several initial huddles
for i in range(options["num_huddles"]):
get_huddle(random.sample(user_profiles_ids, random.randint(3, 4)))
# Create several initial pairs for personals
personals_pairs = [
random.sample(user_profiles_ids, 2)
for i in range(options["num_personals"])
]
# Generate a new set of test data.
create_test_data()
# prepopulate the URL preview/embed data for the links present
# in the config.generate_data.json data set. This makes it
# possible for populate_db to run happily without Internet
# access.
with open("zerver/tests/fixtures/docs_url_preview_data.json",
"r") as f:
urls_with_preview_data = ujson.load(f)
for url in urls_with_preview_data:
cache_set(url, urls_with_preview_data[url], PREVIEW_CACHE_NAME)
threads = options["threads"]
jobs = [
] # type: List[Tuple[int, List[List[int]], Dict[str, Any], Callable[[str], int], int]]
for i in range(threads):
count = options["num_messages"] // threads
if i < options["num_messages"] % threads:
count += 1
jobs.append((count, personals_pairs, options, self.stdout.write,
random.randint(0, 10**10)))
for job in jobs:
generate_and_send_messages(job)
if options["delete"]:
if options["test_suite"]:
# Create test users; the MIT ones are needed to test
# the Zephyr mirroring codepaths.
testsuite_mit_users = [
("Fred Sipb (MIT)", "*****@*****.**"),
("Athena Consulting Exchange User (MIT)",
"*****@*****.**"),
("Esp Classroom (MIT)", "*****@*****.**"),
]
create_users(mit_realm,
testsuite_mit_users,
tos_version=settings.TOS_VERSION)
testsuite_lear_users = [
("King Lear", "*****@*****.**"),
("Cordelia Lear", "*****@*****.**"),
]
create_users(lear_realm,
testsuite_lear_users,
tos_version=settings.TOS_VERSION)
if not options["test_suite"]:
# To keep the messages.json fixtures file for the test
# suite fast, don't add these users and subscriptions
# when running populate_db for the test suite
zulip_stream_dict = {
"devel": {
"description": "For developing"
},
"all": {
"description": "For **everything**"
},
"announce": {
"description": "For announcements",
'stream_post_policy': Stream.STREAM_POST_POLICY_ADMINS
},
"design": {
"description": "For design"
},
"support": {
"description": "For support"
},
"social": {
"description": "For socializing"
},
"test": {
"description": "For testing `code`"
},
"errors": {
"description": "For errors"
},
"sales": {
"description": "For sales discussion"
}
} # type: Dict[str, Dict[str, Any]]
# Calculate the maximum number of digits in any extra stream's
# number, since a stream with name "Extra Stream 3" could show
# up after "Extra Stream 29". (Used later to pad numbers with
# 0s).
maximum_digits = len(str(options['extra_streams'] - 1))
for i in range(options['extra_streams']):
# Pad the number with 0s based on `maximum_digits`.
number_str = str(i).zfill(maximum_digits)
extra_stream_name = 'Extra Stream ' + number_str
zulip_stream_dict[extra_stream_name] = {
"description": "Auto-generated extra stream.",
}
bulk_create_streams(zulip_realm, zulip_stream_dict)
# Now that we've created the notifications stream, configure it properly.
zulip_realm.notifications_stream = get_stream(
"announce", zulip_realm)
zulip_realm.save(update_fields=['notifications_stream'])
# Add a few default streams
for default_stream_name in [
"design", "devel", "social", "support"
]:
DefaultStream.objects.create(realm=zulip_realm,
stream=get_stream(
default_stream_name,
zulip_realm))
# Now subscribe everyone to these streams
subscribe_users_to_streams(zulip_realm, zulip_stream_dict)
# These bots are not needed by the test suite
internal_zulip_users_nosubs = [
("Zulip Commit Bot", "*****@*****.**"),
("Zulip Trac Bot", "*****@*****.**"),
("Zulip Nagios Bot", "*****@*****.**"),
]
create_users(zulip_realm,
internal_zulip_users_nosubs,
bot_type=UserProfile.DEFAULT_BOT)
# Mark all messages as read
UserMessage.objects.all().update(flags=UserMessage.flags.read)
if not options["test_suite"]:
# Update pointer of each user to point to the last message in their
# UserMessage rows with sender_id=user_profile_id.
users = list(
UserMessage.objects.filter(message__sender_id=F(
'user_profile_id')).values('user_profile_id').annotate(
pointer=Max('message_id')))
for user in users:
UserProfile.objects.filter(
id=user['user_profile_id']).update(
pointer=user['pointer'])
create_user_groups()
if not options["test_suite"]:
# We populate the analytics database here for
# development purpose only
call_command('populate_analytics_db')
self.stdout.write("Successfully populated test database.\n")
def test_topics_history(self) -> None:
# verified: int(UserMessage.flags.read) == 1
user_profile = self.example_user("iago")
self.login_user(user_profile)
stream_name = "Verona"
stream = get_stream(stream_name, user_profile.realm)
recipient = stream.recipient
def create_test_message(topic: str) -> int:
# TODO: Clean this up to send messages the normal way.
hamlet = self.example_user("hamlet")
message = Message(
sender=hamlet,
recipient=recipient,
content="whatever",
date_sent=timezone_now(),
sending_client=get_client("whatever"),
)
message.set_topic_name(topic)
message.save()
UserMessage.objects.create(
user_profile=user_profile,
message=message,
flags=0,
)
return message.id
# our most recent topics are topic0, topic1, topic2
# Create old messages with strange spellings.
create_test_message("topic2")
create_test_message("toPIc1")
create_test_message("toPIc0")
create_test_message("topic2")
create_test_message("topic2")
create_test_message("Topic2")
# Create new messages
topic2_msg_id = create_test_message("topic2")
create_test_message("topic1")
create_test_message("topic1")
topic1_msg_id = create_test_message("topic1")
topic0_msg_id = create_test_message("topic0")
endpoint = f"/json/users/me/{stream.id}/topics"
result = self.client_get(endpoint, {})
self.assert_json_success(result)
history = result.json()["topics"]
# We only look at the most recent three topics, because
# the prior fixture data may be unreliable.
history = history[:3]
self.assertEqual(
[topic["name"] for topic in history],
[
"topic0",
"topic1",
"topic2",
],
)
self.assertEqual(
[topic["max_id"] for topic in history],
[
topic0_msg_id,
topic1_msg_id,
topic2_msg_id,
],
)
# Now try as cordelia, who we imagine as a totally new user in
# that she doesn't have UserMessage rows. We should see the
# same results for a public stream.
self.login("cordelia")
result = self.client_get(endpoint, {})
self.assert_json_success(result)
history = result.json()["topics"]
# We only look at the most recent three topics, because
# the prior fixture data may be unreliable.
history = history[:3]
self.assertEqual(
[topic["name"] for topic in history],
[
"topic0",
"topic1",
"topic2",
],
)
self.assertIn("topic0", [topic["name"] for topic in history])
self.assertEqual(
[topic["max_id"] for topic in history],
[
topic0_msg_id,
topic1_msg_id,
topic2_msg_id,
],
)
# Now make stream private, but subscribe cordelia
do_change_stream_permission(stream,
invite_only=True,
acting_user=self.example_user("cordelia"))
self.subscribe(self.example_user("cordelia"), stream.name)
result = self.client_get(endpoint, {})
self.assert_json_success(result)
history = result.json()["topics"]
history = history[:3]
# Cordelia doesn't have these recent history items when we
# wasn't subscribed in her results.
self.assertNotIn("topic0", [topic["name"] for topic in history])
self.assertNotIn("topic1", [topic["name"] for topic in history])
self.assertNotIn("topic2", [topic["name"] for topic in history])
def get_subscription(stream_name: str, user_profile: UserProfile) -> Subscription:
stream = get_stream(stream_name, user_profile.realm)
recipient_id = stream.recipient_id
return Subscription.objects.get(
user_profile=user_profile, recipient_id=recipient_id, active=True
)
def test_raw_unread_stream(self) -> None:
cordelia = self.example_user("cordelia")
hamlet = self.example_user("hamlet")
realm = hamlet.realm
for stream_name in ["social", "devel", "test here"]:
self.subscribe(hamlet, stream_name)
self.subscribe(cordelia, stream_name)
all_message_ids: Set[int] = set()
message_ids = {}
tups = [
("social", "lunch"),
("test here", "bla"),
("devel", "python"),
("devel", "ruby"),
]
for stream_name, topic_name in tups:
message_ids[topic_name] = [
self.send_stream_message(
sender=cordelia,
stream_name=stream_name,
topic_name=topic_name,
)
for i in range(3)
]
all_message_ids |= set(message_ids[topic_name])
self.assert_length(all_message_ids, 12) # sanity check on test setup
self.mute_stream(
user_profile=hamlet,
stream=get_stream("test here", realm),
)
self.mute_topic(
user_profile=hamlet,
stream_name="devel",
topic_name="ruby",
)
raw_unread_data = get_raw_unread_data(
user_profile=hamlet,
)
stream_dict = raw_unread_data["stream_dict"]
self.assertEqual(
set(stream_dict.keys()),
all_message_ids,
)
self.assertEqual(
raw_unread_data["unmuted_stream_msgs"],
set(message_ids["python"]) | set(message_ids["lunch"]),
)
self.assertEqual(
stream_dict[message_ids["lunch"][0]],
dict(
stream_id=get_stream("social", realm).id,
topic="lunch",
),
)
def test_get_realms_and_streams_for_archiving(self) -> None:
zulip_realm = get_realm("zulip")
zulip_realm.message_retention_days = 10
zulip_realm.save()
verona = get_stream("Verona", zulip_realm)
verona.message_retention_days = -1 # Block archiving for this stream
verona.save()
denmark = get_stream("Denmark", zulip_realm)
denmark.message_retention_days = 1
denmark.save()
zephyr_realm = get_realm("zephyr")
zephyr_realm.message_retention_days = -1
zephyr_realm.save()
self.make_stream("normal stream", realm=zephyr_realm)
archiving_blocked_zephyr_stream = self.make_stream("no archiving",
realm=zephyr_realm)
archiving_blocked_zephyr_stream.message_retention_days = -1
archiving_blocked_zephyr_stream.save()
archiving_enabled_zephyr_stream = self.make_stream("with archiving",
realm=zephyr_realm)
archiving_enabled_zephyr_stream.message_retention_days = 1
archiving_enabled_zephyr_stream.save()
no_archiving_realm = do_create_realm(string_id="no_archiving",
name="no_archiving")
do_set_realm_property(no_archiving_realm,
"invite_required",
False,
acting_user=None)
do_set_realm_property(no_archiving_realm,
"message_retention_days",
-1,
acting_user=None)
# Realm for testing the edge case where it has a default retention policy,
# but all streams disable it.
realm_all_streams_archiving_disabled = do_create_realm(
string_id="with_archiving", name="with_archiving")
do_set_realm_property(realm_all_streams_archiving_disabled,
"invite_required",
False,
acting_user=None)
do_set_realm_property(realm_all_streams_archiving_disabled,
"message_retention_days",
1,
acting_user=None)
Stream.objects.filter(
realm=realm_all_streams_archiving_disabled).update(
message_retention_days=-1)
# We construct a list representing how the result of get_realms_and_streams_for_archiving should be.
# One nuisance is that the ordering of the elements in the result structure is not deterministic,
# so we use a helper to order both structures in a consistent manner. This wouldn't be necessary
# if python had a true "unordered list" data structure. Set doesn't do the job, because it requires
# elements to be hashable.
expected_result = [
(zulip_realm,
list(
Stream.objects.filter(realm=zulip_realm).exclude(
id=verona.id))),
(zephyr_realm, [archiving_enabled_zephyr_stream]),
(realm_all_streams_archiving_disabled, []),
]
self.fix_ordering_of_result(expected_result)
simple_algorithm_result = self.simple_get_realms_and_streams_for_archiving(
)
self.fix_ordering_of_result(simple_algorithm_result)
result = get_realms_and_streams_for_archiving()
self.fix_ordering_of_result(result)
self.assert_length(result, len(expected_result))
self.assertEqual(result, expected_result)
self.assert_length(result, len(simple_algorithm_result))
self.assertEqual(result, simple_algorithm_result)
def test_unread_msgs(self) -> None:
sender = self.example_user("cordelia")
sender_id = sender.id
user_profile = self.example_user("hamlet")
othello = self.example_user("othello")
pm1_message_id = self.send_personal_message(sender, user_profile, "hello1")
pm2_message_id = self.send_personal_message(sender, user_profile, "hello2")
muted_stream = self.subscribe(user_profile, "Muted stream")
self.mute_stream(user_profile, muted_stream)
self.mute_topic(user_profile, "Denmark", "muted-topic")
stream_message_id = self.send_stream_message(sender, "Denmark", "hello")
muted_stream_message_id = self.send_stream_message(sender, "Muted stream", "hello")
muted_topic_message_id = self.send_stream_message(
sender,
"Denmark",
topic_name="muted-topic",
content="hello",
)
huddle_message_id = self.send_huddle_message(
sender,
[user_profile, othello],
"hello3",
)
def get_unread_data() -> UnreadMessagesResult:
raw_unread_data = get_raw_unread_data(user_profile)
aggregated_data = aggregate_unread_data(raw_unread_data)
return aggregated_data
with mock.patch("zerver.lib.message.MAX_UNREAD_MESSAGES", 4):
result = get_unread_data()
self.assertEqual(result["count"], 2)
self.assertTrue(result["old_unreads_missing"])
result = get_unread_data()
# The count here reflects the count of unread messages that we will
# report to users in the bankruptcy dialog, and for now it excludes unread messages
# from muted treams, but it doesn't exclude unread messages from muted topics yet.
self.assertEqual(result["count"], 4)
self.assertFalse(result["old_unreads_missing"])
unread_pm = result["pms"][0]
self.assertEqual(unread_pm["sender_id"], sender_id)
self.assertEqual(unread_pm["unread_message_ids"], [pm1_message_id, pm2_message_id])
unread_stream = result["streams"][0]
self.assertEqual(unread_stream["stream_id"], get_stream("Denmark", user_profile.realm).id)
self.assertEqual(unread_stream["topic"], "muted-topic")
self.assertEqual(unread_stream["unread_message_ids"], [muted_topic_message_id])
unread_stream = result["streams"][1]
self.assertEqual(unread_stream["stream_id"], get_stream("Denmark", user_profile.realm).id)
self.assertEqual(unread_stream["topic"], "test")
self.assertEqual(unread_stream["unread_message_ids"], [stream_message_id])
unread_stream = result["streams"][2]
self.assertEqual(
unread_stream["stream_id"], get_stream("Muted stream", user_profile.realm).id
)
self.assertEqual(unread_stream["topic"], "test")
self.assertEqual(unread_stream["unread_message_ids"], [muted_stream_message_id])
huddle_string = ",".join(
str(uid) for uid in sorted([sender_id, user_profile.id, othello.id])
)
unread_huddle = result["huddles"][0]
self.assertEqual(unread_huddle["user_ids_string"], huddle_string)
self.assertEqual(unread_huddle["unread_message_ids"], [huddle_message_id])
self.assertEqual(result["mentions"], [])
um = UserMessage.objects.get(
user_profile_id=user_profile.id,
message_id=stream_message_id,
)
um.flags |= UserMessage.flags.mentioned
um.save()
result = get_unread_data()
self.assertEqual(result["mentions"], [stream_message_id])
um.flags = UserMessage.flags.has_alert_word
um.save()
result = get_unread_data()
# TODO: This should change when we make alert words work better.
self.assertEqual(result["mentions"], [])
um.flags = UserMessage.flags.wildcard_mentioned
um.save()
result = get_unread_data()
self.assertEqual(result["mentions"], [stream_message_id])
um.flags = 0
um.save()
result = get_unread_data()
self.assertEqual(result["mentions"], [])
# Test with a muted stream
um = UserMessage.objects.get(
user_profile_id=user_profile.id,
message_id=muted_stream_message_id,
)
um.flags = UserMessage.flags.mentioned
um.save()
result = get_unread_data()
self.assertEqual(result["mentions"], [muted_stream_message_id])
um.flags = UserMessage.flags.has_alert_word
um.save()
result = get_unread_data()
self.assertEqual(result["mentions"], [])
um.flags = UserMessage.flags.wildcard_mentioned
um.save()
result = get_unread_data()
self.assertEqual(result["mentions"], [])
um.flags = 0
um.save()
result = get_unread_data()
self.assertEqual(result["mentions"], [])
# Test with a muted topic
um = UserMessage.objects.get(
user_profile_id=user_profile.id,
message_id=muted_topic_message_id,
)
um.flags = UserMessage.flags.mentioned
um.save()
result = get_unread_data()
self.assertEqual(result["mentions"], [muted_topic_message_id])
um.flags = UserMessage.flags.has_alert_word
um.save()
result = get_unread_data()
self.assertEqual(result["mentions"], [])
um.flags = UserMessage.flags.wildcard_mentioned
um.save()
result = get_unread_data()
self.assertEqual(result["mentions"], [])
um.flags = 0
um.save()
result = get_unread_data()
self.assertEqual(result["mentions"], [])
