def test_num_queries_with_streams(self) -> None:
main_user = self.example_user('hamlet')
other_user = self.example_user('cordelia')
realm_id = main_user.realm_id
self.login(main_user.email)
# Try to make page-load do extra work for various subscribed
# streams.
for i in range(10):
stream_name = 'test_stream_' + str(i)
stream = self.make_stream(stream_name)
DefaultStream.objects.create(
realm_id=realm_id,
stream_id=stream.id
)
for user in [main_user, other_user]:
self.subscribe(user, stream_name)
# Simulate hitting the page the first time to avoid some noise
# related to initial logins.
self._get_home_page()
# Then for the second page load, measure the number of queries.
flush_per_request_caches()
with queries_captured() as queries2:
result = self._get_home_page()
self.assert_length(queries2, 35)
# Do a sanity check that our new streams were in the payload.
html = result.content.decode('utf-8')
self.assertIn('test_stream_7', html)
def test_num_queries_for_realm_admin(self) -> None:
# Verify number of queries for Realm admin isn't much higher than for normal users.
self.login(self.example_email("iago"))
flush_per_request_caches()
with queries_captured() as queries:
with patch('zerver.lib.cache.cache_set') as cache_mock:
result = self._get_home_page()
self.assertEqual(result.status_code, 200)
self.assert_length(cache_mock.call_args_list, 6)
self.assert_length(queries, 37)
def test_multiple_stream_senders(self,
mock_send_future_email: mock.MagicMock,
mock_enough_traffic: mock.MagicMock) -> None:
client = 'website' # this makes `sent_by_human` return True
othello = self.example_user('othello')
self.subscribe(othello, 'Verona')
one_day_ago = timezone_now() - datetime.timedelta(days=1)
Message.objects.all().update(pub_date=one_day_ago)
one_sec_ago = timezone_now() - datetime.timedelta(seconds=1)
cutoff = time.mktime(one_sec_ago.timetuple())
senders = ['hamlet', 'cordelia', 'iago', 'prospero', 'ZOE']
for sender_name in senders:
email = self.example_email(sender_name)
self.login(email)
content = 'some content for ' + email
payload = dict(
type='stream',
client=client,
to='Verona',
topic='lunch',
content=content,
)
result = self.client_post("/json/messages", payload)
self.assert_json_success(result)
flush_per_request_caches()
with queries_captured() as queries:
handle_digest_email(othello.id, cutoff)
self.assertTrue(24 <= len(queries) <= 25)
self.assertEqual(mock_send_future_email.call_count, 1)
kwargs = mock_send_future_email.call_args[1]
self.assertEqual(kwargs['to_user_ids'], [othello.id])
hot_convo = kwargs['context']['hot_conversations'][0]
expected_participants = {
self.example_user(sender).full_name
for sender in senders
}
self.assertEqual(set(hot_convo['participants']), expected_participants)
self.assertEqual(hot_convo['count'], 5 - 2) # 5 messages, but 2 shown
teaser_messages = hot_convo['first_few_messages'][0]['senders']
self.assertIn('some content', teaser_messages[0]['content'][0]['plain'])
self.assertIn(teaser_messages[0]['sender'], expected_participants)
def test_activity(self) -> None:
self.login(self.example_email("hamlet"))
client, _ = Client.objects.get_or_create(name='website')
query = '/json/users/me/pointer'
last_visit = timezone_now()
count = 150
for user_profile in UserProfile.objects.all():
UserActivity.objects.get_or_create(
user_profile=user_profile,
client=client,
query=query,
count=count,
last_visit=last_visit
)
flush_per_request_caches()
with queries_captured() as queries:
self.client_get('/activity')
self.assert_length(queries, 4)
def test_activity(self) -> None:
self.login(self.example_email("hamlet"))
client, _ = Client.objects.get_or_create(name='website')
query = '/json/users/me/pointer'
last_visit = timezone_now()
count = 150
for activity_user_profile in UserProfile.objects.all():
UserActivity.objects.get_or_create(
user_profile=activity_user_profile,
client=client,
query=query,
count=count,
last_visit=last_visit
)
# Fails when not staff
result = self.client_get('/activity')
self.assertEqual(result.status_code, 302)
user_profile = self.example_user("hamlet")
user_profile.is_staff = True
user_profile.save()
flush_per_request_caches()
with queries_captured() as queries:
result = self.client_get('/activity')
self.assertEqual(result.status_code, 200)
self.assert_length(queries, 13)
flush_per_request_caches()
with queries_captured() as queries:
result = self.client_get('/realm_activity/zulip/')
self.assertEqual(result.status_code, 200)
self.assert_length(queries, 9)
flush_per_request_caches()
with queries_captured() as queries:
result = self.client_get('/user_activity/[email protected]/')
self.assertEqual(result.status_code, 200)
self.assert_length(queries, 5)
def test_home(self) -> None:
# Keep this list sorted!!!
html_bits = [
'Compose your message here...',
'Exclude messages with topic',
'Keyboard shortcuts',
'Loading...',
'Manage streams',
'Narrow to topic',
'Next message',
'Search streams',
'Welcome to Zulip',
# Verify that the app styles get included
'app-stubentry.js',
'data-params',
]
# Keep this list sorted!!!
expected_keys = [
"alert_words",
"available_notification_sounds",
"avatar_source",
"avatar_url",
"avatar_url_medium",
"bot_types",
"can_create_streams",
"can_subscribe_other_users",
"cross_realm_bots",
"custom_profile_field_types",
"custom_profile_fields",
"debug_mode",
"default_language",
"default_language_name",
"delivery_email",
"demote_inactive_streams",
"dense_mode",
"desktop_icon_count_display",
"development_environment",
"email",
"emojiset",
"emojiset_choices",
"enable_desktop_notifications",
"enable_digest_emails",
"enable_login_emails",
"enable_offline_email_notifications",
"enable_offline_push_notifications",
"enable_online_push_notifications",
"enable_sounds",
"enable_stream_audible_notifications",
"enable_stream_desktop_notifications",
"enable_stream_email_notifications",
"enable_stream_push_notifications",
"enter_sends",
"first_in_realm",
"fluid_layout_width",
"full_name",
"furthest_read_time",
"has_mobile_devices",
"have_initial_messages",
"high_contrast_mode",
"hotspots",
"initial_servertime",
"insecure_desktop_app",
"is_admin",
"is_guest",
"jitsi_server_url",
"language_list",
"language_list_dbl_col",
"last_event_id",
"left_side_userlist",
"login_page",
"max_avatar_file_size",
"max_file_upload_size",
"max_icon_file_size",
"max_logo_file_size",
"max_message_id",
"message_content_in_email_notifications",
"muted_topics",
"narrow",
"narrow_stream",
"needs_tutorial",
"never_subscribed",
"night_mode",
"notification_sound",
"password_min_guesses",
"password_min_length",
"plan_includes_wide_organization_logo",
"pm_content_in_desktop_notifications",
"pointer",
"poll_timeout",
"presences",
"prompt_for_invites",
"queue_id",
"realm_add_emoji_by_admins_only",
"realm_allow_community_topic_editing",
"realm_allow_edit_history",
"realm_allow_message_deleting",
"realm_allow_message_editing",
"realm_authentication_methods",
"realm_available_video_chat_providers",
"realm_avatar_changes_disabled",
"realm_bot_creation_policy",
"realm_bot_domain",
"realm_bots",
"realm_create_stream_policy",
"realm_default_code_block_language",
"realm_default_external_accounts",
"realm_default_language",
"realm_default_stream_groups",
"realm_default_streams",
"realm_default_twenty_four_hour_time",
"realm_description",
"realm_digest_emails_enabled",
"realm_digest_weekday",
"realm_disallow_disposable_email_addresses",
"realm_domains",
"realm_email_address_visibility",
"realm_email_auth_enabled",
"realm_email_changes_disabled",
"realm_emails_restricted_to_domains",
"realm_embedded_bots",
"realm_emoji",
"realm_filters",
"realm_google_hangouts_domain",
"realm_icon_source",
"realm_icon_url",
"realm_incoming_webhook_bots",
"realm_inline_image_preview",
"realm_inline_url_embed_preview",
"realm_invite_by_admins_only",
"realm_invite_required",
"realm_invite_to_stream_policy",
"realm_is_zephyr_mirror_realm",
"realm_logo_source",
"realm_logo_url",
"realm_mandatory_topics",
"realm_message_content_allowed_in_email_notifications",
"realm_message_content_delete_limit_seconds",
"realm_message_content_edit_limit_seconds",
"realm_message_retention_days",
"realm_name",
"realm_name_changes_disabled",
"realm_name_in_notifications",
"realm_night_logo_source",
"realm_night_logo_url",
"realm_non_active_users",
"realm_notifications_stream_id",
"realm_password_auth_enabled",
"realm_plan_type",
"realm_presence_disabled",
"realm_private_message_policy",
"realm_push_notifications_enabled",
"realm_send_welcome_emails",
"realm_signup_notifications_stream_id",
"realm_upload_quota",
"realm_uri",
"realm_user_group_edit_policy",
"realm_user_groups",
"realm_users",
"realm_video_chat_provider",
"realm_waiting_period_threshold",
"realm_zoom_api_key",
"realm_zoom_api_secret",
"realm_zoom_user_id",
"recent_private_conversations",
"root_domain_uri",
"save_stacktraces",
"search_pills_enabled",
"server_avatar_changes_disabled",
"server_generation",
"server_inline_image_preview",
"server_inline_url_embed_preview",
"server_name_changes_disabled",
"settings_send_digest_emails",
"starred_message_counts",
"starred_messages",
"stop_words",
"stream_description_max_length",
"stream_name_max_length",
"subscriptions",
"test_suite",
"timezone",
"translate_emoticons",
"translation_data",
"twenty_four_hour_time",
"two_fa_enabled",
"two_fa_enabled_user",
"unread_msgs",
"unsubscribed",
"upgrade_text_for_wide_organization_logo",
"user_id",
"user_status",
"warn_no_email",
"webpack_public_path",
"wildcard_mentions_notify",
"zulip_feature_level",
"zulip_version",
]
# Verify fails if logged-out
result = self.client_get('/')
self.assertEqual(result.status_code, 302)
self.login('hamlet')
# Create bot for realm_bots testing. Must be done before fetching home_page.
bot_info = {
'full_name': 'The Bot of Hamlet',
'short_name': 'hambot',
}
self.client_post("/json/bots", bot_info)
# Verify succeeds once logged-in
flush_per_request_caches()
with queries_captured() as queries:
with patch('zerver.lib.cache.cache_set') as cache_mock:
result = self._get_home_page(stream='Denmark')
self.assertEqual(set(result["Cache-Control"].split(", ")),
{"must-revalidate", "no-store", "no-cache"})
self.assert_length(queries, 43)
self.assert_length(cache_mock.call_args_list, 5)
html = result.content.decode('utf-8')
for html_bit in html_bits:
if html_bit not in html:
raise AssertionError('%s not in result' % (html_bit, ))
page_params = self._get_page_params(result)
actual_keys = sorted([str(k) for k in page_params.keys()])
self.assertEqual(actual_keys, expected_keys)
# TODO: Inspect the page_params data further.
# print(ujson.dumps(page_params, indent=2))
realm_bots_expected_keys = [
'api_key',
'avatar_url',
'bot_type',
'default_all_public_streams',
'default_events_register_stream',
'default_sending_stream',
'email',
'full_name',
'is_active',
'owner',
'services',
'user_id',
]
realm_bots_actual_keys = sorted(
[str(key) for key in page_params['realm_bots'][0].keys()])
self.assertEqual(realm_bots_actual_keys, realm_bots_expected_keys)
def test_queries(self) -> None:
user = self.example_user("hamlet")
self.login_user(user)
flush_per_request_caches()
with queries_captured() as queries:
with mock.patch("zerver.lib.events.always_want") as want_mock:
fetch_initial_state_data(user)
self.assert_length(queries, 31)
expected_counts = dict(
alert_words=1,
custom_profile_fields=1,
default_streams=1,
default_stream_groups=1,
hotspots=0,
message=1,
muted_topics=1,
muted_users=1,
presence=1,
realm=0,
realm_bot=1,
realm_domains=1,
realm_embedded_bots=0,
realm_incoming_webhook_bots=0,
realm_emoji=1,
realm_filters=1,
realm_linkifiers=1,
realm_playgrounds=1,
realm_user=3,
realm_user_groups=2,
recent_private_conversations=1,
starred_messages=1,
stream=2,
stop_words=0,
subscription=4,
update_display_settings=0,
update_global_notifications=0,
update_message_flags=5,
user_status=1,
video_calls=0,
giphy=0,
)
wanted_event_types = {item[0][0] for item in want_mock.call_args_list}
self.assertEqual(wanted_event_types, set(expected_counts))
for event_type in sorted(wanted_event_types):
count = expected_counts[event_type]
flush_per_request_caches()
with queries_captured() as queries:
if event_type == "update_message_flags":
event_types = ["update_message_flags", "message"]
else:
event_types = [event_type]
fetch_initial_state_data(user, event_types=event_types)
self.assert_length(queries, count)
def test_soft_deactivated_user_multiple_stream_senders(self) -> None:
one_day_ago = timezone_now() - datetime.timedelta(days=1)
Message.objects.all().update(date_sent=one_day_ago)
digest_users = [
self.example_user("othello"),
self.example_user("aaron"),
self.example_user("desdemona"),
self.example_user("polonius"),
]
digest_users.sort(key=lambda user: user.id)
for digest_user in digest_users:
for stream in ["Verona", "Scotland", "Denmark"]:
self.subscribe(digest_user, stream)
RealmAuditLog.objects.all().delete()
# Send messages to a stream and unsubscribe - subscribe from that stream
senders = ["hamlet", "cordelia", "iago", "prospero", "ZOE"]
self.simulate_stream_conversation("Verona", senders)
for digest_user in digest_users:
self.unsubscribe(digest_user, "Verona")
self.subscribe(digest_user, "Verona")
# Send messages to other streams
self.simulate_stream_conversation("Scotland", senders)
self.simulate_stream_conversation("Denmark", senders)
one_hour_ago = timezone_now() - datetime.timedelta(seconds=3600)
cutoff = time.mktime(one_hour_ago.timetuple())
flush_per_request_caches()
# When this test is run in isolation, one additional query is run which
# is equivalent to
# ContentType.objects.get(app_label='zerver', model='userprofile')
# This code is run when we call `confirmation.models.create_confirmation_link`.
# To trigger this, we call the one_click_unsubscribe_link function below.
one_click_unsubscribe_link(digest_users[0], "digest")
with mock.patch("zerver.lib.digest.send_future_email") as mock_send_future_email:
digest_user_ids = [user.id for user in digest_users]
with queries_captured() as queries:
with cache_tries_captured() as cache_tries:
bulk_handle_digest_email(digest_user_ids, cutoff)
self.assert_length(queries, 12)
self.assert_length(cache_tries, 0)
self.assertEqual(mock_send_future_email.call_count, len(digest_users))
for i, digest_user in enumerate(digest_users):
kwargs = mock_send_future_email.call_args_list[i][1]
self.assertEqual(kwargs["to_user_ids"], [digest_user.id])
hot_conversations = kwargs["context"]["hot_conversations"]
self.assertEqual(2, len(hot_conversations), [digest_user.id])
hot_convo = hot_conversations[0]
expected_participants = {self.example_user(sender).full_name for sender in senders}
self.assertEqual(set(hot_convo["participants"]), expected_participants)
self.assertEqual(hot_convo["count"], 5 - 2) # 5 messages, but 2 shown
teaser_messages = hot_convo["first_few_messages"][0]["senders"]
self.assertIn("some content", teaser_messages[0]["content"][0]["plain"])
self.assertIn(teaser_messages[0]["sender"], expected_participants)
last_message_id = get_last_message_id()
for digest_user in digest_users:
log_rows = RealmAuditLog.objects.filter(
modified_user_id=digest_user.id,
event_type=RealmAuditLog.USER_DIGEST_EMAIL_CREATED,
)
(log,) = log_rows
self.assertEqual(log.event_last_message_id, last_message_id)
def do_consume(
self, consume_func: Callable[[List[Dict[str, Any]]], None], events: List[Dict[str, Any]]
) -> None:
consume_time_seconds: Optional[float] = None
with configure_scope() as scope:
scope.clear_breadcrumbs()
add_breadcrumb(
type="debug",
category="queue_processor",
message=f"Consuming {self.queue_name}",
data={"events": events, "local_queue_size": self.get_remaining_local_queue_size()},
)
try:
if self.idle:
# We're reactivating after having gone idle due to emptying the queue.
# We should update the stats file to keep it fresh and to make it clear
# that the queue started processing, in case the event we're about to process
# makes us freeze.
self.idle = False
self.update_statistics(self.get_remaining_local_queue_size())
time_start = time.time()
if self.MAX_CONSUME_SECONDS and self.ENABLE_TIMEOUTS:
try:
signal.signal(
signal.SIGALRM,
functools.partial(timer_expired, self.MAX_CONSUME_SECONDS, len(events)),
)
try:
signal.alarm(self.MAX_CONSUME_SECONDS * len(events))
consume_func(events)
finally:
signal.alarm(0)
finally:
signal.signal(signal.SIGALRM, signal.SIG_DFL)
else:
consume_func(events)
consume_time_seconds = time.time() - time_start
self.consumed_since_last_emptied += len(events)
except Exception as e:
self._handle_consume_exception(events, e)
finally:
flush_per_request_caches()
reset_queries()
if consume_time_seconds is not None:
self.recent_consume_times.append((len(events), consume_time_seconds))
remaining_local_queue_size = self.get_remaining_local_queue_size()
if remaining_local_queue_size == 0:
self.queue_last_emptied_timestamp = time.time()
self.consumed_since_last_emptied = 0
# We've cleared all the events from the queue, so we don't
# need to worry about the small overhead of doing a disk write.
# We take advantage of this to update the stats file to keep it fresh,
# especially since the queue might go idle until new events come in.
self.update_statistics(0)
self.idle = True
return
self.consume_iteration_counter += 1
if (
self.consume_iteration_counter >= self.CONSUME_ITERATIONS_BEFORE_UPDATE_STATS_NUM
or time.time() - self.last_statistics_update_time
>= self.MAX_SECONDS_BEFORE_UPDATE_STATS
):
self.consume_iteration_counter = 0
self.update_statistics(remaining_local_queue_size)
def process_response(self, request: HttpRequest,
response: HttpResponse) -> HttpResponse:
# We flush the per-request caches after every request, so they
# are not shared at all between requests.
flush_per_request_caches()
return response
def test_home(self) -> None:
# Keep this list sorted!!!
html_bits = [
'Compose your message here...',
'Exclude messages with topic',
'Keyboard shortcuts',
'Loading...',
'Manage streams',
'Narrow by topic',
'Next message',
'Search streams',
'Welcome to Zulip',
'pygments.css',
'var page_params',
]
# Keep this list sorted!!!
expected_keys = [
"alert_words",
"attachments",
"autoscroll_forever",
"avatar_source",
"avatar_url",
"avatar_url_medium",
"can_create_streams",
"cross_realm_bots",
"custom_profile_fields",
"debug_mode",
"default_desktop_notifications",
"default_language",
"default_language_name",
"development_environment",
"email",
"emoji_alt_code",
"emojiset",
"emojiset_choices",
"enable_desktop_notifications",
"enable_digest_emails",
"enable_offline_email_notifications",
"enable_offline_push_notifications",
"enable_online_push_notifications",
"enable_sounds",
"enable_stream_desktop_notifications",
"enable_stream_email_notifications",
"enable_stream_push_notifications",
"enable_stream_sounds",
"enter_sends",
"first_in_realm",
"full_name",
"furthest_read_time",
"has_mobile_devices",
"have_initial_messages",
"high_contrast_mode",
"hotspots",
"initial_servertime",
"is_admin",
"language_list",
"language_list_dbl_col",
"last_event_id",
"left_side_userlist",
"login_page",
"max_avatar_file_size",
"max_icon_file_size",
"max_message_id",
"maxfilesize",
"muted_topics",
"narrow",
"narrow_stream",
"needs_tutorial",
"never_subscribed",
"night_mode",
"password_min_guesses",
"password_min_length",
"pm_content_in_desktop_notifications",
"pointer",
"poll_timeout",
"presences",
"prompt_for_invites",
"queue_id",
"realm_add_emoji_by_admins_only",
"realm_allow_edit_history",
"realm_allow_message_deleting",
"realm_allow_message_editing",
"realm_authentication_methods",
"realm_bot_domain",
"realm_bots",
"realm_create_stream_by_admins_only",
"realm_default_language",
"realm_default_stream_groups",
"realm_default_streams",
"realm_description",
"realm_domains",
"realm_email_auth_enabled",
"realm_email_changes_disabled",
"realm_embedded_bots",
"realm_emoji",
"realm_filters",
"realm_icon_source",
"realm_icon_url",
"realm_inline_image_preview",
"realm_inline_url_embed_preview",
"realm_invite_by_admins_only",
"realm_invite_required",
"realm_is_zephyr_mirror_realm",
"realm_mandatory_topics",
"realm_message_content_edit_limit_seconds",
"realm_message_retention_days",
"realm_name",
"realm_name_changes_disabled",
"realm_non_active_users",
"realm_notifications_stream_id",
"realm_password_auth_enabled",
"realm_presence_disabled",
"realm_restricted_to_domain",
"realm_show_digest_email",
"realm_signup_notifications_stream_id",
"realm_uri",
"realm_user_groups",
"realm_users",
"realm_waiting_period_threshold",
"root_domain_uri",
"save_stacktraces",
"server_generation",
"server_inline_image_preview",
"server_inline_url_embed_preview",
"subscriptions",
"test_suite",
"timezone",
"total_uploads_size",
"twenty_four_hour_time",
"unread_msgs",
"unsubscribed",
"upload_quota",
"use_websockets",
"user_id",
"zulip_version",
]
email = self.example_email("hamlet")
# Verify fails if logged-out
result = self.client_get('/')
self.assertEqual(result.status_code, 302)
self.login(email)
# Create bot for realm_bots testing. Must be done before fetching home_page.
bot_info = {
'full_name': 'The Bot of Hamlet',
'short_name': 'hambot',
}
self.client_post("/json/bots", bot_info)
# Verify succeeds once logged-in
flush_per_request_caches()
with queries_captured() as queries:
with patch('zerver.lib.cache.cache_set') as cache_mock:
result = self._get_home_page(stream='Denmark')
self.assert_length(queries, 40)
self.assert_length(cache_mock.call_args_list, 7)
html = result.content.decode('utf-8')
for html_bit in html_bits:
if html_bit not in html:
raise AssertionError('%s not in result' % (html_bit, ))
page_params = self._get_page_params(result)
actual_keys = sorted([str(k) for k in page_params.keys()])
self.assertEqual(actual_keys, expected_keys)
# TODO: Inspect the page_params data further.
# print(ujson.dumps(page_params, indent=2))
realm_bots_expected_keys = [
'api_key',
'avatar_url',
'bot_type',
'default_all_public_streams',
'default_events_register_stream',
'default_sending_stream',
'email',
'full_name',
'is_active',
'owner',
'user_id',
]
realm_bots_actual_keys = sorted(
[str(key) for key in page_params['realm_bots'][0].keys()])
self.assertEqual(realm_bots_actual_keys, realm_bots_expected_keys)
def test_soft_deactivated_user_multiple_stream_senders(self) -> None:
one_day_ago = timezone_now() - datetime.timedelta(days=1)
Message.objects.all().update(date_sent=one_day_ago)
digest_users = [
self.example_user('othello'),
self.example_user('aaron'),
self.example_user('desdemona'),
self.example_user('polonius'),
]
for digest_user in digest_users:
for stream in ['Verona', 'Scotland', 'Denmark']:
self.subscribe(digest_user, stream)
RealmAuditLog.objects.all().delete()
for digest_user in digest_users:
digest_user.long_term_idle = True
digest_user.save(update_fields=['long_term_idle'])
# Send messages to a stream and unsubscribe - subscribe from that stream
senders = ['hamlet', 'cordelia', 'iago', 'prospero', 'ZOE']
self.simulate_stream_conversation('Verona', senders)
for digest_user in digest_users:
self.unsubscribe(digest_user, 'Verona')
self.subscribe(digest_user, 'Verona')
# Send messages to other streams
self.simulate_stream_conversation('Scotland', senders)
self.simulate_stream_conversation('Denmark', senders)
one_hour_ago = timezone_now() - datetime.timedelta(seconds=3600)
cutoff = time.mktime(one_hour_ago.timetuple())
flush_per_request_caches()
# When this test is run in isolation, one additional query is run which
# is equivalent to
# ContentType.objects.get(app_label='zerver', model='userprofile')
# This code is run when we call `confirmation.models.create_confirmation_link`.
# To trigger this, we call the one_click_unsubscribe_link function below.
one_click_unsubscribe_link(digest_users[0], 'digest')
with mock.patch('zerver.lib.digest.send_future_email'
) as mock_send_future_email:
digest_user_ids = [user.id for user in digest_users]
with queries_captured() as queries:
with cache_tries_captured() as cache_tries:
bulk_handle_digest_email(digest_user_ids, cutoff)
self.assert_length(queries, 37)
self.assert_length(cache_tries, 4)
self.assertEqual(mock_send_future_email.call_count, len(digest_users))
for i, digest_user in enumerate(digest_users):
kwargs = mock_send_future_email.call_args_list[i][1]
self.assertEqual(kwargs['to_user_ids'], [digest_user.id])
hot_conversations = kwargs['context']['hot_conversations']
self.assertEqual(2, len(hot_conversations), [digest_user.id])
hot_convo = hot_conversations[0]
expected_participants = {
self.example_user(sender).full_name
for sender in senders
}
self.assertEqual(set(hot_convo['participants']),
expected_participants)
self.assertEqual(hot_convo['count'],
5 - 2) # 5 messages, but 2 shown
teaser_messages = hot_convo['first_few_messages'][0]['senders']
self.assertIn('some content',
teaser_messages[0]['content'][0]['plain'])
self.assertIn(teaser_messages[0]['sender'], expected_participants)
def test_soft_deactivated_user_multiple_stream_senders(self,
mock_send_future_email: mock.MagicMock,
mock_enough_traffic: mock.MagicMock) -> None:
one_day_ago = timezone_now() - datetime.timedelta(days=1)
Message.objects.all().update(pub_date=one_day_ago)
othello = self.example_user('othello')
for stream in ['Verona', 'Scotland', 'Denmark']:
self.subscribe(othello, stream)
RealmAuditLog.objects.all().delete()
othello.long_term_idle = True
othello.save(update_fields=['long_term_idle'])
# Send messages to a stream and unsubscribe - subscribe from that stream
senders = ['hamlet', 'cordelia', 'iago', 'prospero', 'ZOE']
self.simulate_stream_conversation('Verona', senders)
self.unsubscribe(othello, 'Verona')
self.subscribe(othello, 'Verona')
# Send messages to other streams
self.simulate_stream_conversation('Scotland', senders)
self.simulate_stream_conversation('Denmark', senders)
one_hour_ago = timezone_now() - datetime.timedelta(seconds=3600)
cutoff = time.mktime(one_hour_ago.timetuple())
flush_per_request_caches()
# When this test is run in isolation, one additional query is run which
# is equivalent to
# ContentType.objects.get(app_label='zerver', model='userprofile')
# This code is run when we call `confirmation.models.create_confirmation_link`.
# To trigger this, we call the one_click_unsubscribe_link function below.
one_click_unsubscribe_link(othello, 'digest')
with queries_captured() as queries:
handle_digest_email(othello.id, cutoff)
# This can definitely be optimized; for both the huddle and
# stream cases, the get_narrow_url API ends up double-fetching
# some data because of how the functions are organized.
self.assert_length(queries, 9)
self.assertEqual(mock_send_future_email.call_count, 1)
kwargs = mock_send_future_email.call_args[1]
self.assertEqual(kwargs['to_user_ids'], [othello.id])
hot_conversations = kwargs['context']['hot_conversations']
self.assertEqual(2, len(hot_conversations), [othello.id])
hot_convo = hot_conversations[0]
expected_participants = {
self.example_user(sender).full_name
for sender in senders
}
self.assertEqual(set(hot_convo['participants']), expected_participants)
self.assertEqual(hot_convo['count'], 5 - 2) # 5 messages, but 2 shown
teaser_messages = hot_convo['first_few_messages'][0]['senders']
self.assertIn('some content', teaser_messages[0]['content'][0]['plain'])
self.assertIn(teaser_messages[0]['sender'], expected_participants)
def test_home(self) -> None:
# Keep this list sorted!!!
html_bits = [
'Compose your message here...',
'Exclude messages with topic',
'Keyboard shortcuts',
'Loading...',
'Manage streams',
'Narrow to topic',
'Next message',
'Search streams',
# Verify that the app styles get included
'app-stubentry.js',
'data-params',
]
# Verify fails if logged-out
result = self.client_get('/')
self.assertEqual(result.status_code, 302)
self.login('hamlet')
# Create bot for realm_bots testing. Must be done before fetching home_page.
bot_info = {
'full_name': 'The Bot of Hamlet',
'short_name': 'hambot',
}
self.client_post("/json/bots", bot_info)
# Verify succeeds once logged-in
flush_per_request_caches()
with queries_captured() as queries:
with patch('zerver.lib.cache.cache_set') as cache_mock:
result = self._get_home_page(stream='Denmark')
self.check_rendered_logged_in_app(result)
self.assertEqual(set(result["Cache-Control"].split(", ")),
{"must-revalidate", "no-store", "no-cache"})
self.assert_length(queries, 42)
self.assert_length(cache_mock.call_args_list, 5)
html = result.content.decode('utf-8')
for html_bit in html_bits:
if html_bit not in html:
raise AssertionError(f'{html_bit} not in result')
page_params = self._get_page_params(result)
actual_keys = sorted(str(k) for k in page_params.keys())
self.assertEqual(actual_keys, self.expected_page_params_keys)
# TODO: Inspect the page_params data further.
# print(orjson.dumps(page_params, option=orjson.OPT_INDENT_2).decode())
realm_bots_expected_keys = [
'api_key',
'avatar_url',
'bot_type',
'default_all_public_streams',
'default_events_register_stream',
'default_sending_stream',
'email',
'full_name',
'is_active',
'owner_id',
'services',
'user_id',
]
realm_bots_actual_keys = sorted(str(key) for key in page_params['realm_bots'][0].keys())
self.assertEqual(realm_bots_actual_keys, realm_bots_expected_keys)
def test_home(self) -> None:
# Keep this list sorted!!!
html_bits = [
'Compose your message here...',
'Exclude messages with topic',
'Keyboard shortcuts',
'Loading...',
'Manage streams',
'Narrow by topic',
'Next message',
'Search streams',
'Welcome to Zulip',
# Verify that the app styles get included
'app-styles-stubentry.js',
'var page_params',
]
# Keep this list sorted!!!
expected_keys = [
"alert_words",
"avatar_source",
"avatar_url",
"avatar_url_medium",
"bot_types",
"can_create_streams",
"cross_realm_bots",
"custom_profile_field_types",
"custom_profile_fields",
"debug_mode",
"default_language",
"default_language_name",
"development_environment",
"email",
"emojiset",
"emojiset_choices",
"enable_desktop_notifications",
"enable_digest_emails",
"enable_offline_email_notifications",
"enable_offline_push_notifications",
"enable_online_push_notifications",
"enable_sounds",
"enable_stream_desktop_notifications",
"enable_stream_email_notifications",
"enable_stream_push_notifications",
"enable_stream_sounds",
"enter_sends",
"first_in_realm",
"full_name",
"furthest_read_time",
"has_mobile_devices",
"have_initial_messages",
"high_contrast_mode",
"hotspots",
"initial_servertime",
"is_admin",
"jitsi_server_url",
"language_list",
"language_list_dbl_col",
"last_event_id",
"left_side_userlist",
"login_page",
"max_avatar_file_size",
"max_icon_file_size",
"max_message_id",
"maxfilesize",
"message_content_in_email_notifications",
"muted_topics",
"narrow",
"narrow_stream",
"needs_tutorial",
"never_subscribed",
"night_mode",
"password_min_guesses",
"password_min_length",
"pm_content_in_desktop_notifications",
"pointer",
"poll_timeout",
"presences",
"prompt_for_invites",
"queue_id",
"realm_add_emoji_by_admins_only",
"realm_allow_community_topic_editing",
"realm_allow_edit_history",
"realm_allow_message_deleting",
"realm_allow_message_editing",
"realm_authentication_methods",
"realm_available_video_chat_providers",
"realm_bot_creation_policy",
"realm_bot_domain",
"realm_bots",
"realm_create_stream_by_admins_only",
"realm_default_language",
"realm_default_stream_groups",
"realm_default_streams",
"realm_default_twenty_four_hour_time",
"realm_description",
"realm_disallow_disposable_email_addresses",
"realm_domains",
"realm_email_auth_enabled",
"realm_email_changes_disabled",
"realm_embedded_bots",
"realm_emoji",
"realm_filters",
"realm_google_hangouts_domain",
"realm_icon_source",
"realm_icon_url",
"realm_inline_image_preview",
"realm_inline_url_embed_preview",
"realm_invite_by_admins_only",
"realm_invite_required",
"realm_is_zephyr_mirror_realm",
"realm_mandatory_topics",
"realm_message_content_delete_limit_seconds",
"realm_message_content_edit_limit_seconds",
"realm_message_retention_days",
"realm_name",
"realm_name_changes_disabled",
"realm_name_in_notifications",
"realm_non_active_users",
"realm_notifications_stream_id",
"realm_password_auth_enabled",
"realm_presence_disabled",
"realm_push_notifications_enabled",
"realm_restricted_to_domain",
"realm_send_welcome_emails",
"realm_show_digest_email",
"realm_signup_notifications_stream_id",
"realm_uri",
"realm_user_groups",
"realm_users",
"realm_video_chat_provider",
"realm_waiting_period_threshold",
"root_domain_uri",
"save_stacktraces",
"server_generation",
"server_inline_image_preview",
"server_inline_url_embed_preview",
"stream_description_max_length",
"stream_name_max_length",
"subscriptions",
"test_suite",
"timezone",
"translate_emoticons",
"twenty_four_hour_time",
"unread_msgs",
"unsubscribed",
"use_websockets",
"user_id",
"warn_no_email",
"zulip_version",
]
email = self.example_email("hamlet")
# Verify fails if logged-out
result = self.client_get('/')
self.assertEqual(result.status_code, 302)
self.login(email)
# Create bot for realm_bots testing. Must be done before fetching home_page.
bot_info = {
'full_name': 'The Bot of Hamlet',
'short_name': 'hambot',
}
self.client_post("/json/bots", bot_info)
# Verify succeeds once logged-in
flush_per_request_caches()
with queries_captured() as queries:
with patch('zerver.lib.cache.cache_set') as cache_mock:
result = self._get_home_page(stream='Denmark')
self.assert_length(queries, 41)
self.assert_length(cache_mock.call_args_list, 7)
html = result.content.decode('utf-8')
for html_bit in html_bits:
if html_bit not in html:
raise AssertionError('%s not in result' % (html_bit,))
page_params = self._get_page_params(result)
actual_keys = sorted([str(k) for k in page_params.keys()])
self.assertEqual(actual_keys, expected_keys)
# TODO: Inspect the page_params data further.
# print(ujson.dumps(page_params, indent=2))
realm_bots_expected_keys = [
'api_key',
'avatar_url',
'bot_type',
'default_all_public_streams',
'default_events_register_stream',
'default_sending_stream',
'email',
'full_name',
'is_active',
'owner',
'services',
'user_id',
]
realm_bots_actual_keys = sorted([str(key) for key in page_params['realm_bots'][0].keys()])
self.assertEqual(realm_bots_actual_keys, realm_bots_expected_keys)
def test_home(self) -> None:
# Keep this list sorted!!!
html_bits = [
"start the conversation",
"Keyboard shortcuts",
"Loading...",
"Filter streams",
# Verify that the app styles get included
"app-stubentry.js",
"data-params",
]
self.login("hamlet")
# Create bot for realm_bots testing. Must be done before fetching home_page.
bot_info = {
"full_name": "The Bot of Hamlet",
"short_name": "hambot",
}
self.client_post("/json/bots", bot_info)
# Verify succeeds once logged-in
flush_per_request_caches()
with queries_captured() as queries:
with patch("zerver.lib.cache.cache_set") as cache_mock:
result = self._get_home_page(stream="Denmark")
self.check_rendered_logged_in_app(result)
self.assertEqual(set(result["Cache-Control"].split(", ")),
{"must-revalidate", "no-store", "no-cache"})
self.assert_length(queries, 40)
self.assert_length(cache_mock.call_args_list, 5)
html = result.content.decode("utf-8")
for html_bit in html_bits:
if html_bit not in html:
raise AssertionError(f"{html_bit} not in result")
page_params = self._get_page_params(result)
actual_keys = sorted(str(k) for k in page_params.keys())
self.assertEqual(actual_keys, self.expected_page_params_keys)
# TODO: Inspect the page_params data further.
# print(orjson.dumps(page_params, option=orjson.OPT_INDENT_2).decode())
realm_bots_expected_keys = [
"api_key",
"avatar_url",
"bot_type",
"default_all_public_streams",
"default_events_register_stream",
"default_sending_stream",
"email",
"full_name",
"is_active",
"owner_id",
"services",
"user_id",
]
realm_bots_actual_keys = sorted(
str(key) for key in page_params["realm_bots"][0].keys())
self.assertEqual(realm_bots_actual_keys, realm_bots_expected_keys)
def process_response(self, request, response):
# type: (HttpRequest, HttpResponse) -> HttpResponse
# We flush the per-request caches after every request, so they
# are not shared at all between requests.
flush_per_request_caches()
return response
def tearDown(self) -> None:
super().tearDown()
# Important: we need to clear event queues to avoid leaking data to future tests.
clear_client_event_queues_for_testing()
clear_supported_auth_backends_cache()
flush_per_request_caches()