def func_with_caching(*args, **kwargs):
# type: (*Any, **Any) -> Callable[..., Any]
key = keyfunc(*args, **kwargs)
val = cache_get(key, cache_name=cache_name)
extra = ""
if cache_name == 'database':
extra = ".dbcache"
if with_statsd_key is not None:
metric_key = with_statsd_key
else:
metric_key = statsd_key(key)
status = "hit" if val is not None else "miss"
statsd.incr("cache%s.%s.%s" % (extra, metric_key, status))
# Values are singleton tuples so that we can distinguish
# a result of None from a missing key.
if val is not None:
return val[0]
val = func(*args, **kwargs)
cache_set(key, val, cache_name=cache_name, timeout=timeout)
return val
def report_send_times(request: HttpRequest, user_profile: UserProfile,
time: int=REQ(converter=to_non_negative_int),
received: int=REQ(converter=to_non_negative_int, default=-1),
displayed: int=REQ(converter=to_non_negative_int, default=-1),
locally_echoed: bool=REQ(validator=check_bool, default=False),
rendered_content_disparity: bool=REQ(validator=check_bool,
default=False)) -> HttpResponse:
received_str = "(unknown)"
if received > 0:
received_str = str(received)
displayed_str = "(unknown)"
if displayed > 0:
displayed_str = str(displayed)
request._log_data["extra"] = f"[{time}ms/{received_str}ms/{displayed_str}ms/echo:{locally_echoed}/diff:{rendered_content_disparity}]"
base_key = statsd_key(user_profile.realm.string_id, clean_periods=True)
statsd.timing(f"endtoend.send_time.{base_key}", time)
if received > 0:
statsd.timing(f"endtoend.receive_time.{base_key}", received)
if displayed > 0:
statsd.timing(f"endtoend.displayed_time.{base_key}", displayed)
if locally_echoed:
statsd.incr('locally_echoed')
if rendered_content_disparity:
statsd.incr('render_disparity')
return json_success()
def json_report_send_time(request,
user_profile,
time=REQ(converter=to_non_negative_int),
received=REQ(converter=to_non_negative_int,
default="(unknown)"),
displayed=REQ(converter=to_non_negative_int,
default="(unknown)"),
locally_echoed=REQ(validator=check_bool,
default=False),
rendered_content_disparity=REQ(validator=check_bool,
default=False)):
# type: (HttpRequest, UserProfile, int, int, int, bool, bool) -> HttpResponse
request._log_data["extra"] = "[%sms/%sms/%sms/echo:%s/diff:%s]" \
% (time, received, displayed, locally_echoed, rendered_content_disparity)
base_key = statsd_key(user_profile.realm.string_id, clean_periods=True)
statsd.timing("endtoend.send_time.%s" % (base_key, ), time)
if received != "(unknown)":
statsd.timing("endtoend.receive_time.%s" % (base_key, ), received)
if displayed != "(unknown)":
statsd.timing("endtoend.displayed_time.%s" % (base_key, ), displayed)
if locally_echoed:
statsd.incr('locally_echoed')
if rendered_content_disparity:
statsd.incr('render_disparity')
return json_success()
def func_with_caching(*args: object, **kwargs: object) -> object:
key = keyfunc(*args, **kwargs)
try:
val = cache_get(key, cache_name=cache_name)
except InvalidCacheKeyException:
stack_trace = traceback.format_exc()
log_invalid_cache_keys(stack_trace, [key])
return func(*args, **kwargs)
extra = ""
if cache_name == 'database':
extra = ".dbcache"
if with_statsd_key is not None:
metric_key = with_statsd_key
else:
metric_key = statsd_key(key)
status = "hit" if val is not None else "miss"
statsd.incr(f"cache{extra}.{metric_key}.{status}")
# Values are singleton tuples so that we can distinguish
# a result of None from a missing key.
if val is not None:
return val[0]
val = func(*args, **kwargs)
cache_set(key, val, cache_name=cache_name, timeout=timeout)
return val
def report_send_times(request: HttpRequest, user_profile: UserProfile,
time: int=REQ(converter=to_non_negative_int),
received: int=REQ(converter=to_non_negative_int, default=-1),
displayed: int=REQ(converter=to_non_negative_int, default=-1),
locally_echoed: bool=REQ(validator=check_bool, default=False),
rendered_content_disparity: bool=REQ(validator=check_bool,
default=False)) -> HttpResponse:
received_str = "(unknown)"
if received > 0:
received_str = str(received)
displayed_str = "(unknown)"
if displayed > 0:
displayed_str = str(displayed)
request._log_data["extra"] = "[%sms/%sms/%sms/echo:%s/diff:%s]" \
% (time, received_str, displayed_str, locally_echoed, rendered_content_disparity)
base_key = statsd_key(user_profile.realm.string_id, clean_periods=True)
statsd.timing("endtoend.send_time.%s" % (base_key,), time)
if received > 0:
statsd.timing("endtoend.receive_time.%s" % (base_key,), received)
if displayed > 0:
statsd.timing("endtoend.displayed_time.%s" % (base_key,), displayed)
if locally_echoed:
statsd.incr('locally_echoed')
if rendered_content_disparity:
statsd.incr('render_disparity')
return json_success()
def func_with_caching(*args, **kwargs):
# type: (*Any, **Any) -> Callable[..., Any]
key = keyfunc(*args, **kwargs)
val = cache_get(key, cache_name=cache_name)
extra = ""
if cache_name == 'database':
extra = ".dbcache"
if with_statsd_key is not None:
metric_key = with_statsd_key
else:
metric_key = statsd_key(key)
status = "hit" if val is not None else "miss"
statsd.incr("cache%s.%s.%s" % (extra, metric_key, status))
# Values are singleton tuples so that we can distinguish
# a result of None from a missing key.
if val is not None:
return val[0]
val = func(*args, **kwargs)
cache_set(key, val, cache_name=cache_name, timeout=timeout)
return val
def rate_limit_user(request: HttpRequest, user: UserProfile, domain: str) -> None:
"""Returns whether or not a user was rate limited. Will raise a RateLimited exception
if the user has been rate limited, otherwise returns and modifies request to contain
the rate limit information"""
entity = RateLimitedUser(user, domain=domain)
ratelimited, time = is_ratelimited(entity)
request._ratelimit_applied_limits = True
request._ratelimit_secs_to_freedom = time
request._ratelimit_over_limit = ratelimited
# Abort this request if the user is over their rate limits
if ratelimited:
statsd.incr("ratelimiter.limited.%s.%s" % (type(user), user.id))
raise RateLimited()
try:
incr_ratelimit(entity)
except RateLimiterLockingException: # nocoverage # Should add on next rate limit pass
logging.warning("Deadlock trying to incr_ratelimit for %s on %s" % (
user.id, request.path))
# rate-limit users who are hitting the API so hard we can't update our stats.
raise RateLimited()
calls_remaining, time_reset = api_calls_left(entity)
request._ratelimit_remaining = calls_remaining
request._ratelimit_secs_to_freedom = time_reset
def rate_limit_user(request: HttpRequest, user: UserProfile, domain: str) -> None:
"""Returns whether or not a user was rate limited. Will raise a RateLimited exception
if the user has been rate limited, otherwise returns and modifies request to contain
the rate limit information"""
entity = RateLimitedUser(user, domain=domain)
ratelimited, time = is_ratelimited(entity)
request._ratelimit_applied_limits = True
request._ratelimit_secs_to_freedom = time
request._ratelimit_over_limit = ratelimited
# Abort this request if the user is over their rate limits
if ratelimited:
statsd.incr("ratelimiter.limited.%s.%s" % (type(user), user.id))
raise RateLimited()
try:
incr_ratelimit(entity)
except RateLimiterLockingException:
logging.warning("Deadlock trying to incr_ratelimit for %s on %s" % (
user.id, request.path))
# rate-limit users who are hitting the API so hard we can't update our stats.
raise RateLimited()
calls_remaining, time_reset = api_calls_left(entity)
request._ratelimit_remaining = calls_remaining
request._ratelimit_secs_to_freedom = time_reset
def do_publish() -> None:
self.channel.basic_publish(
exchange='',
routing_key=queue_name,
properties=pika.BasicProperties(delivery_mode=2),
body=body)
statsd.incr("rabbitmq.publish.%s" % (queue_name, ))
def do_publish() -> None:
self.channel.basic_publish(
exchange='',
routing_key=queue_name,
properties=pika.BasicProperties(delivery_mode=2),
body=body)
statsd.incr("rabbitmq.publish.%s" % (queue_name,))
def do_publish(channel: BlockingChannel) -> None:
channel.basic_publish(
exchange='',
routing_key=queue_name,
properties=pika.BasicProperties(delivery_mode=2),
body=body)
statsd.incr(f"rabbitmq.publish.{queue_name}")
def rate_limit_entity(cls, entity_key: str, rules: List[Tuple[int, int]],
max_api_calls: int, max_api_window: int) -> Tuple[bool, float]:
ratelimited, time = cls.is_ratelimited(entity_key, rules)
if ratelimited:
statsd.incr("ratelimiter.limited.%s" % (entity_key,))
else:
try:
cls.incr_ratelimit(entity_key, rules, max_api_calls, max_api_window)
except RateLimiterLockingException:
logger.warning("Deadlock trying to incr_ratelimit for %s" % (entity_key,))
# rate-limit users who are hitting the API so hard we can't update our stats.
ratelimited = True
return ratelimited, time
def rate_limit_mirror_by_realm(recipient_realm: Realm) -> None:
# Code based on the rate_limit_user function:
entity = RateLimitedRealmMirror(recipient_realm)
ratelimited, time = is_ratelimited(entity)
if ratelimited:
statsd.incr("ratelimiter.limited.%s.%s" %
(type(recipient_realm), recipient_realm.id))
raise RateLimited()
try:
incr_ratelimit(entity)
except RateLimiterLockingException:
logger.warning("Email mirror rate limiter: Deadlock trying to "
"incr_ratelimit for realm %s" %
(recipient_realm.name, ))
raise RateLimited()
def rate_limit_entity(entity: RateLimitedObject) -> Tuple[bool, float]:
# Returns (ratelimited, secs_to_freedom)
ratelimited, time = is_ratelimited(entity)
if ratelimited:
statsd.incr("ratelimiter.limited.%s.%s" % (type(entity), str(entity)))
else:
try:
incr_ratelimit(entity)
except RateLimiterLockingException:
logger.warning("Deadlock trying to incr_ratelimit for %s:%s" %
(type(entity).__name__, str(entity)))
# rate-limit users who are hitting the API so hard we can't update our stats.
ratelimited = True
return ratelimited, time
def json_report_send_time(request, user_profile,
time=REQ(converter=to_non_negative_int),
received=REQ(converter=to_non_negative_int, default="(unknown)"),
displayed=REQ(converter=to_non_negative_int, default="(unknown)"),
locally_echoed=REQ(validator=check_bool, default=False),
rendered_content_disparity=REQ(validator=check_bool, default=False)):
request._log_data["extra"] = "[%sms/%sms/%sms/echo:%s/diff:%s]" \
% (time, received, displayed, locally_echoed, rendered_content_disparity)
statsd.timing("endtoend.send_time.%s" % (statsd_key(user_profile.realm.domain, clean_periods=True),), time)
if received != "(unknown)":
statsd.timing("endtoend.receive_time.%s" % (statsd_key(user_profile.realm.domain, clean_periods=True),), received)
if displayed != "(unknown)":
statsd.timing("endtoend.displayed_time.%s" % (statsd_key(user_profile.realm.domain, clean_periods=True),), displayed)
if locally_echoed:
statsd.incr('locally_echoed')
if rendered_content_disparity:
statsd.incr('render_disparity')
return json_success()
def rate_limit_user(request, user, domain):
"""Returns whether or not a user was rate limited. Will raise a RateLimited exception
if the user has been rate limited, otherwise returns and modifies request to contain
the rate limit information"""
ratelimited, time = is_ratelimited(user, domain)
request._ratelimit_applied_limits = True
request._ratelimit_secs_to_freedom = time
request._ratelimit_over_limit = ratelimited
# Abort this request if the user is over her rate limits
if ratelimited:
statsd.incr("ratelimiter.limited.%s.%s" % (type(user), user.id))
raise RateLimited()
incr_ratelimit(user, domain)
calls_remaining, time_reset = api_calls_left(user, domain)
request._ratelimit_remaining = calls_remaining
request._ratelimit_secs_to_freedom = time_reset
def rate_limit_user(request, user, domain):
"""Returns whether or not a user was rate limited. Will raise a RateLimited exception
if the user has been rate limited, otherwise returns and modifies request to contain
the rate limit information"""
ratelimited, time = is_ratelimited(user, domain)
request._ratelimit_applied_limits = True
request._ratelimit_secs_to_freedom = time
request._ratelimit_over_limit = ratelimited
# Abort this request if the user is over her rate limits
if ratelimited:
statsd.incr("ratelimiter.limited.%s.%s" % (type(user), user.id))
raise RateLimited()
incr_ratelimit(user, domain)
calls_remaining, time_reset = api_calls_left(user, domain)
request._ratelimit_remaining = calls_remaining
request._ratelimit_secs_to_freedom = time_reset
def rate_limit_entity(cls, entity_key: str, rules: List[Tuple[int, int]],
max_api_calls: int,
max_api_window: int) -> Tuple[bool, float]:
now = time.time()
if entity_key in cls.timestamps_blocked_until:
# Check whether the key is manually blocked.
if now < cls.timestamps_blocked_until[entity_key]:
blocking_ttl = cls.timestamps_blocked_until[entity_key] - now
return True, blocking_ttl
else:
del cls.timestamps_blocked_until[entity_key]
assert rules
for time_window, max_count in rules:
ratelimited, time_till_free = cls.need_to_limit(
entity_key, time_window, max_count)
if ratelimited:
statsd.incr("ratelimiter.limited.%s" % (entity_key, ))
break
return ratelimited, time_till_free
def wrapped_func(*args: object, **kwargs: object) -> object:
ret = func(*args, **kwargs)
statsd.incr(counter, val)
return ret
def home_real(request):
# type: (HttpRequest) -> HttpResponse
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
user_profile = request.user
# If a user hasn't signed the current Terms of Service, send them there
if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \
int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version():
return accounts_accept_terms(request)
narrow = [] # type: List[List[Text]]
narrow_stream = None
narrow_topic = request.GET.get("topic")
if request.GET.get("stream"):
try:
narrow_stream_name = request.GET.get("stream")
(narrow_stream, ignored_rec,
ignored_sub) = access_stream_by_name(user_profile,
narrow_stream_name)
narrow = [["stream", narrow_stream.name]]
except Exception:
logging.exception("Narrow parsing")
if narrow_stream is not None and narrow_topic is not None:
narrow.append(["topic", narrow_topic])
register_ret = do_events_register(user_profile,
request.client,
apply_markdown=True,
narrow=narrow)
user_has_messages = (register_ret['max_message_id'] != -1)
# Reset our don't-spam-users-with-email counter since the
# user has since logged in
if user_profile.last_reminder is not None:
user_profile.last_reminder = None
user_profile.save(update_fields=["last_reminder"])
# Brand new users get the tutorial
needs_tutorial = settings.TUTORIAL_ENABLED and \
user_profile.tutorial_status != UserProfile.TUTORIAL_FINISHED
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = first_in_realm and \
not PreregistrationUser.objects.filter(referred_by=user_profile).count()
if user_profile.pointer == -1 and user_has_messages:
# Put the new user's pointer at the bottom
#
# This improves performance, because we limit backfilling of messages
# before the pointer. It's also likely that someone joining an
# organization is interested in recent messages more than the very
# first messages on the system.
register_ret['pointer'] = register_ret['max_message_id']
user_profile.last_pointer_updater = request.session.session_key
if user_profile.pointer == -1:
latest_read = None
else:
try:
latest_read = UserMessage.objects.get(
user_profile=user_profile, message__id=user_profile.pointer)
except UserMessage.DoesNotExist:
# Don't completely fail if your saved pointer ID is invalid
logging.warning("%s has invalid pointer %s" %
(user_profile.email, user_profile.pointer))
latest_read = None
desktop_notifications_enabled = user_profile.enable_desktop_notifications
if narrow_stream is not None:
desktop_notifications_enabled = False
if user_profile.realm.notifications_stream:
notifications_stream = user_profile.realm.notifications_stream.name
else:
notifications_stream = ""
# Set default language and make it persist
default_language = register_ret['default_language']
url_lang = '/{}'.format(request.LANGUAGE_CODE)
if not request.path.startswith(url_lang):
translation.activate(default_language)
request.session[translation.LANGUAGE_SESSION_KEY] = default_language
# Pass parameters to the client-side JavaScript code.
# These end up in a global JavaScript Object named 'page_params'.
page_params = dict(
# Server settings.
share_the_love=settings.SHARE_THE_LOVE,
development_environment=settings.DEVELOPMENT,
debug_mode=settings.DEBUG,
test_suite=settings.TEST_SUITE,
poll_timeout=settings.POLL_TIMEOUT,
login_page=settings.HOME_NOT_LOGGED_IN,
server_uri=settings.SERVER_URI,
maxfilesize=settings.MAX_FILE_UPLOAD_SIZE,
server_generation=settings.SERVER_GENERATION,
use_websockets=settings.USE_WEBSOCKETS,
save_stacktraces=settings.SAVE_FRONTEND_STACKTRACES,
# realm data.
# TODO: Move all of these data to register_ret and pull from there
realm_uri=user_profile.realm.uri,
password_auth_enabled=password_auth_enabled(user_profile.realm),
domain=user_profile.realm.domain,
domains=list_of_domains_for_realm(user_profile.realm),
realm_icon_url=realm_icon_url(user_profile.realm),
realm_icon_source=user_profile.realm.icon_source,
name_changes_disabled=name_changes_disabled(user_profile.realm),
mandatory_topics=user_profile.realm.mandatory_topics,
show_digest_email=user_profile.realm.show_digest_email,
realm_presence_disabled=user_profile.realm.presence_disabled,
is_zephyr_mirror_realm=user_profile.realm.is_zephyr_mirror_realm,
# user_profile data.
# TODO: Move all of these data to register_ret and pull from there
fullname=user_profile.full_name,
email=user_profile.email,
enter_sends=user_profile.enter_sends,
user_id=user_profile.id,
is_admin=user_profile.is_realm_admin,
can_create_streams=user_profile.can_create_streams(),
autoscroll_forever=user_profile.autoscroll_forever,
default_desktop_notifications=user_profile.
default_desktop_notifications,
avatar_url=avatar_url(user_profile),
avatar_url_medium=avatar_url(user_profile, medium=True),
avatar_source=user_profile.avatar_source,
# Stream message notification settings:
stream_desktop_notifications_enabled=user_profile.
enable_stream_desktop_notifications,
stream_sounds_enabled=user_profile.enable_stream_sounds,
# Private message and @-mention notification settings:
desktop_notifications_enabled=desktop_notifications_enabled,
sounds_enabled=user_profile.enable_sounds,
enable_offline_email_notifications=user_profile.
enable_offline_email_notifications,
pm_content_in_desktop_notifications=user_profile.
pm_content_in_desktop_notifications,
enable_offline_push_notifications=user_profile.
enable_offline_push_notifications,
enable_online_push_notifications=user_profile.
enable_online_push_notifications,
enable_digest_emails=user_profile.enable_digest_emails,
# Realm foreign key data from register_ret.
# TODO: Rename these to match register_ret values.
subbed_info=register_ret['subscriptions'],
unsubbed_info=register_ret['unsubscribed'],
neversubbed_info=register_ret['never_subscribed'],
people_list=register_ret['realm_users'],
bot_list=register_ret['realm_bots'],
initial_pointer=register_ret['pointer'],
initial_presences=register_ret['presences'],
event_queue_id=register_ret['queue_id'],
# Misc. extra data.
have_initial_messages=user_has_messages,
initial_servertime=time.time(
), # Used for calculating relative presence age
default_language_name=get_language_name(
register_ret['default_language']),
language_list_dbl_col=get_language_list_for_templates(
register_ret['default_language']),
language_list=get_language_list(),
needs_tutorial=needs_tutorial,
first_in_realm=first_in_realm,
prompt_for_invites=prompt_for_invites,
notifications_stream=notifications_stream,
cross_realm_bots=list(get_cross_realm_dicts()),
unread_count=approximate_unread_count(user_profile),
furthest_read_time=sent_time_in_epoch_seconds(latest_read),
has_mobile_devices=num_push_devices_for_user(user_profile) > 0,
)
# These fields will be automatically copied from register_ret into
# page_params. It is a goal to move more of the page_params list
# into this sort of cleaner structure.
page_params_core_fields = [
'alert_words',
'attachments',
'default_language',
'last_event_id',
'left_side_userlist',
'max_message_id',
'muted_topics',
'realm_add_emoji_by_admins_only',
'realm_allow_message_editing',
'realm_authentication_methods',
'realm_create_stream_by_admins_only',
'realm_default_language',
'realm_default_streams',
'realm_emoji',
'realm_message_content_edit_limit_seconds',
'realm_name',
'realm_invite_by_admins_only',
'realm_invite_required',
'realm_filters',
'realm_restricted_to_domain',
'realm_waiting_period_threshold',
'referrals',
'twenty_four_hour_time',
'zulip_version',
]
for field_name in page_params_core_fields:
page_params[field_name] = register_ret[field_name]
if narrow_stream is not None:
# In narrow_stream context, initial pointer is just latest message
recipient = get_recipient(Recipient.STREAM, narrow_stream.id)
try:
initial_pointer = Message.objects.filter(
recipient=recipient).order_by('id').reverse()[0].id
except IndexError:
initial_pointer = -1
page_params["narrow_stream"] = narrow_stream.name
if narrow_topic is not None:
page_params["narrow_topic"] = narrow_topic
page_params["narrow"] = [
dict(operator=term[0], operand=term[1]) for term in narrow
]
page_params["max_message_id"] = initial_pointer
page_params["initial_pointer"] = initial_pointer
page_params["have_initial_messages"] = (initial_pointer != -1)
statsd.incr('views.home')
show_invites = True
# Some realms only allow admins to invite users
if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin:
show_invites = False
product_name = "Zulip"
page_params['product_name'] = product_name
request._log_data['extra'] = "[%s]" % (register_ret["queue_id"], )
response = render_to_response('zerver/index.html', {
'user_profile':
user_profile,
'page_params':
simplejson.encoder.JSONEncoderForHTML().encode(page_params),
'nofontface':
is_buggy_ua(request.META.get("HTTP_USER_AGENT", "Unspecified")),
'avatar_url':
avatar_url(user_profile),
'show_debug':
settings.DEBUG and ('show_debug' in request.GET),
'pipeline':
settings.PIPELINE_ENABLED,
'show_invites':
show_invites,
'is_admin':
user_profile.is_realm_admin,
'show_webathena':
user_profile.realm.webathena_enabled,
'enable_feedback':
settings.ENABLE_FEEDBACK,
'embedded':
narrow_stream is not None,
'product_name':
product_name
},
request=request)
patch_cache_control(response,
no_cache=True,
no_store=True,
must_revalidate=True)
return response
def write_log_line(
log_data: MutableMapping[str, Any],
path: str,
method: str,
remote_ip: str,
requestor_for_logs: str,
client_name: str,
status_code: int = 200,
error_content: Optional[AnyStr] = None,
error_content_iter: Optional[Iterable[AnyStr]] = None) -> None:
assert error_content is None or error_content_iter is None
if error_content is not None:
error_content_iter = (error_content, )
if settings.STATSD_HOST != '':
# For statsd timer name
if path == '/':
statsd_path = 'webreq'
else:
statsd_path = "webreq.{}".format(path[1:].replace('/', '.'))
# Remove non-ascii chars from path (there should be none, if there are it's
# because someone manually entered a nonexistent path), as UTF-8 chars make
# statsd sad when it sends the key name over the socket
statsd_path = statsd_path.encode('ascii',
errors='ignore').decode("ascii")
# TODO: This could probably be optimized to use a regular expression rather than a loop.
suppress_statsd = any(blacklisted in statsd_path
for blacklisted in statsd_blacklisted_requests)
else:
suppress_statsd = True
statsd_path = ''
time_delta = -1
# A time duration of -1 means the StartLogRequests middleware
# didn't run for some reason
optional_orig_delta = ""
if 'time_started' in log_data:
time_delta = time.time() - log_data['time_started']
if 'time_stopped' in log_data:
orig_time_delta = time_delta
time_delta = ((log_data['time_stopped'] - log_data['time_started']) +
(time.time() - log_data['time_restarted']))
optional_orig_delta = f" (lp: {format_timedelta(orig_time_delta)})"
remote_cache_output = ""
if 'remote_cache_time_start' in log_data:
remote_cache_time_delta = get_remote_cache_time(
) - log_data['remote_cache_time_start']
remote_cache_count_delta = get_remote_cache_requests(
) - log_data['remote_cache_requests_start']
if 'remote_cache_requests_stopped' in log_data:
# (now - restarted) + (stopped - start) = (now - start) + (stopped - restarted)
remote_cache_time_delta += (
log_data['remote_cache_time_stopped'] -
log_data['remote_cache_time_restarted'])
remote_cache_count_delta += (
log_data['remote_cache_requests_stopped'] -
log_data['remote_cache_requests_restarted'])
if (remote_cache_time_delta > 0.005):
remote_cache_output = f" (mem: {format_timedelta(remote_cache_time_delta)}/{remote_cache_count_delta})"
if not suppress_statsd:
statsd.timing(f"{statsd_path}.remote_cache.time",
timedelta_ms(remote_cache_time_delta))
statsd.incr(f"{statsd_path}.remote_cache.querycount",
remote_cache_count_delta)
startup_output = ""
if 'startup_time_delta' in log_data and log_data[
"startup_time_delta"] > 0.005:
startup_output = " (+start: {})".format(
format_timedelta(log_data["startup_time_delta"]))
markdown_output = ""
if 'markdown_time_start' in log_data:
markdown_time_delta = get_markdown_time(
) - log_data['markdown_time_start']
markdown_count_delta = get_markdown_requests(
) - log_data['markdown_requests_start']
if 'markdown_requests_stopped' in log_data:
# (now - restarted) + (stopped - start) = (now - start) + (stopped - restarted)
markdown_time_delta += (log_data['markdown_time_stopped'] -
log_data['markdown_time_restarted'])
markdown_count_delta += (log_data['markdown_requests_stopped'] -
log_data['markdown_requests_restarted'])
if (markdown_time_delta > 0.005):
markdown_output = f" (md: {format_timedelta(markdown_time_delta)}/{markdown_count_delta})"
if not suppress_statsd:
statsd.timing(f"{statsd_path}.markdown.time",
timedelta_ms(markdown_time_delta))
statsd.incr(f"{statsd_path}.markdown.count",
markdown_count_delta)
# Get the amount of time spent doing database queries
db_time_output = ""
queries = connection.connection.queries if connection.connection is not None else []
if len(queries) > 0:
query_time = sum(float(query.get('time', 0)) for query in queries)
db_time_output = f" (db: {format_timedelta(query_time)}/{len(queries)}q)"
if not suppress_statsd:
# Log ms, db ms, and num queries to statsd
statsd.timing(f"{statsd_path}.dbtime", timedelta_ms(query_time))
statsd.incr(f"{statsd_path}.dbq", len(queries))
statsd.timing(f"{statsd_path}.total", timedelta_ms(time_delta))
if 'extra' in log_data:
extra_request_data = " {}".format(log_data['extra'])
else:
extra_request_data = ""
logger_client = f"({requestor_for_logs} via {client_name})"
logger_timing = f'{format_timedelta(time_delta):>5}{optional_orig_delta}{remote_cache_output}{markdown_output}{db_time_output}{startup_output} {path}'
logger_line = f'{remote_ip:<15} {method:<7} {status_code:3} {logger_timing}{extra_request_data} {logger_client}'
if (status_code in [200, 304] and method == "GET"
and path.startswith("/static")):
logger.debug(logger_line)
else:
logger.info(logger_line)
if (is_slow_query(time_delta, path)):
slow_query_logger.info(logger_line)
if settings.PROFILE_ALL_REQUESTS:
log_data["prof"].disable()
profile_path = "/tmp/profile.data.{}.{}".format(
path.split("/")[-1], int(time_delta * 1000))
log_data["prof"].dump_stats(profile_path)
# Log some additional data whenever we return certain 40x errors
if 400 <= status_code < 500 and status_code not in [401, 404, 405]:
assert error_content_iter is not None
error_content_list = list(error_content_iter)
if not error_content_list:
error_data = ''
elif isinstance(error_content_list[0], str):
error_data = ''.join(error_content_list)
elif isinstance(error_content_list[0], bytes):
error_data = repr(b''.join(error_content_list))
if len(error_data) > 200:
error_data = "[content more than 200 characters]"
logger.info('status=%3d, data=%s, uid=%s', status_code, error_data,
requestor_for_logs)
def home_real(request):
# type: (HttpRequest) -> HttpResponse
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
user_profile = request.user
# If a user hasn't signed the current Terms of Service, send them there
if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \
int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version():
return accounts_accept_terms(request)
narrow = [] # type: List[List[Text]]
narrow_stream = None
narrow_topic = request.GET.get("topic")
if request.GET.get("stream"):
try:
narrow_stream_name = request.GET.get("stream")
(narrow_stream, ignored_rec, ignored_sub) = access_stream_by_name(
user_profile, narrow_stream_name)
narrow = [["stream", narrow_stream.name]]
except Exception:
logging.exception("Narrow parsing")
if narrow_stream is not None and narrow_topic is not None:
narrow.append(["topic", narrow_topic])
register_ret = do_events_register(user_profile, request.client,
apply_markdown=True, narrow=narrow)
user_has_messages = (register_ret['max_message_id'] != -1)
# Reset our don't-spam-users-with-email counter since the
# user has since logged in
if user_profile.last_reminder is not None:
user_profile.last_reminder = None
user_profile.save(update_fields=["last_reminder"])
# Brand new users get narrowed to PM with welcome-bot
needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = first_in_realm and \
not PreregistrationUser.objects.filter(referred_by=user_profile).count()
if user_profile.pointer == -1 and user_has_messages:
# Put the new user's pointer at the bottom
#
# This improves performance, because we limit backfilling of messages
# before the pointer. It's also likely that someone joining an
# organization is interested in recent messages more than the very
# first messages on the system.
register_ret['pointer'] = register_ret['max_message_id']
user_profile.last_pointer_updater = request.session.session_key
if user_profile.pointer == -1:
latest_read = None
else:
try:
latest_read = UserMessage.objects.get(user_profile=user_profile,
message__id=user_profile.pointer)
except UserMessage.DoesNotExist:
# Don't completely fail if your saved pointer ID is invalid
logging.warning("%s has invalid pointer %s" % (user_profile.email, user_profile.pointer))
latest_read = None
# Set default language and make it persist
default_language = register_ret['default_language']
url_lang = '/{}'.format(request.LANGUAGE_CODE)
if not request.path.startswith(url_lang):
translation.activate(default_language)
request.session[translation.LANGUAGE_SESSION_KEY] = default_language
# Pass parameters to the client-side JavaScript code.
# These end up in a global JavaScript Object named 'page_params'.
page_params = dict(
# Server settings.
development_environment = settings.DEVELOPMENT,
debug_mode = settings.DEBUG,
test_suite = settings.TEST_SUITE,
poll_timeout = settings.POLL_TIMEOUT,
login_page = settings.HOME_NOT_LOGGED_IN,
root_domain_uri = settings.ROOT_DOMAIN_URI,
maxfilesize = settings.MAX_FILE_UPLOAD_SIZE,
max_avatar_file_size = settings.MAX_AVATAR_FILE_SIZE,
server_generation = settings.SERVER_GENERATION,
use_websockets = settings.USE_WEBSOCKETS,
save_stacktraces = settings.SAVE_FRONTEND_STACKTRACES,
server_inline_image_preview = settings.INLINE_IMAGE_PREVIEW,
server_inline_url_embed_preview = settings.INLINE_URL_EMBED_PREVIEW,
password_min_length = settings.PASSWORD_MIN_LENGTH,
password_min_guesses = settings.PASSWORD_MIN_GUESSES,
# Misc. extra data.
have_initial_messages = user_has_messages,
initial_servertime = time.time(), # Used for calculating relative presence age
default_language_name = get_language_name(register_ret['default_language']),
language_list_dbl_col = get_language_list_for_templates(register_ret['default_language']),
language_list = get_language_list(),
needs_tutorial = needs_tutorial,
first_in_realm = first_in_realm,
prompt_for_invites = prompt_for_invites,
furthest_read_time = sent_time_in_epoch_seconds(latest_read),
has_mobile_devices = num_push_devices_for_user(user_profile) > 0,
)
undesired_register_ret_fields = [
'streams',
]
for field_name in set(register_ret.keys()) - set(undesired_register_ret_fields):
page_params[field_name] = register_ret[field_name]
if narrow_stream is not None:
# In narrow_stream context, initial pointer is just latest message
recipient = get_recipient(Recipient.STREAM, narrow_stream.id)
try:
initial_pointer = Message.objects.filter(recipient=recipient).order_by('id').reverse()[0].id
except IndexError:
initial_pointer = -1
page_params["narrow_stream"] = narrow_stream.name
if narrow_topic is not None:
page_params["narrow_topic"] = narrow_topic
page_params["narrow"] = [dict(operator=term[0], operand=term[1]) for term in narrow]
page_params["max_message_id"] = initial_pointer
page_params["pointer"] = initial_pointer
page_params["have_initial_messages"] = (initial_pointer != -1)
page_params["enable_desktop_notifications"] = False
statsd.incr('views.home')
show_invites = True
# Some realms only allow admins to invite users
if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin:
show_invites = False
request._log_data['extra'] = "[%s]" % (register_ret["queue_id"],)
response = render(request, 'zerver/index.html',
context={'user_profile': user_profile,
'page_params': simplejson.encoder.JSONEncoderForHTML().encode(page_params),
'nofontface': is_buggy_ua(request.META.get("HTTP_USER_AGENT", "Unspecified")),
'avatar_url': avatar_url(user_profile),
'show_debug':
settings.DEBUG and ('show_debug' in request.GET),
'pipeline': settings.PIPELINE_ENABLED,
'show_invites': show_invites,
'is_admin': user_profile.is_realm_admin,
'show_webathena': user_profile.realm.webathena_enabled,
'enable_feedback': settings.ENABLE_FEEDBACK,
'embedded': narrow_stream is not None,
},)
patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True)
return response
def home(request):
# type: (HttpRequest) -> HttpResponse
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
user_profile = request.user
request._email = request.user.email
request.client = get_client("website")
# If a user hasn't signed the current Terms of Service, send them there
if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \
int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version():
return accounts_accept_terms(request)
narrow = [] # type: List[List[text_type]]
narrow_stream = None
narrow_topic = request.GET.get("topic")
if request.GET.get("stream"):
try:
narrow_stream = get_stream(request.GET.get("stream"), user_profile.realm)
assert(narrow_stream is not None)
assert(narrow_stream.is_public())
narrow = [["stream", narrow_stream.name]]
except Exception:
logging.exception("Narrow parsing")
if narrow_topic is not None:
narrow.append(["topic", narrow_topic])
register_ret = do_events_register(user_profile, request.client,
apply_markdown=True, narrow=narrow)
user_has_messages = (register_ret['max_message_id'] != -1)
# Reset our don't-spam-users-with-email counter since the
# user has since logged in
if not user_profile.last_reminder is None:
user_profile.last_reminder = None
user_profile.save(update_fields=["last_reminder"])
# Brand new users get the tutorial
needs_tutorial = settings.TUTORIAL_ENABLED and \
user_profile.tutorial_status != UserProfile.TUTORIAL_FINISHED
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = first_in_realm and \
not PreregistrationUser.objects.filter(referred_by=user_profile).count()
if user_profile.pointer == -1 and user_has_messages:
# Put the new user's pointer at the bottom
#
# This improves performance, because we limit backfilling of messages
# before the pointer. It's also likely that someone joining an
# organization is interested in recent messages more than the very
# first messages on the system.
register_ret['pointer'] = register_ret['max_message_id']
user_profile.last_pointer_updater = request.session.session_key
if user_profile.pointer == -1:
latest_read = None
else:
try:
latest_read = UserMessage.objects.get(user_profile=user_profile,
message__id=user_profile.pointer)
except UserMessage.DoesNotExist:
# Don't completely fail if your saved pointer ID is invalid
logging.warning("%s has invalid pointer %s" % (user_profile.email, user_profile.pointer))
latest_read = None
desktop_notifications_enabled = user_profile.enable_desktop_notifications
if narrow_stream is not None:
desktop_notifications_enabled = False
if user_profile.realm.notifications_stream:
notifications_stream = user_profile.realm.notifications_stream.name
else:
notifications_stream = ""
# Set default language and make it persist
default_language = register_ret['default_language']
url_lang = '/{}'.format(request.LANGUAGE_CODE)
if not request.path.startswith(url_lang):
translation.activate(default_language)
request.session[translation.LANGUAGE_SESSION_KEY] = default_language
# Pass parameters to the client-side JavaScript code.
# These end up in a global JavaScript Object named 'page_params'.
page_params = dict(
zulip_version = ZULIP_VERSION,
share_the_love = settings.SHARE_THE_LOVE,
development_environment = settings.DEVELOPMENT,
debug_mode = settings.DEBUG,
test_suite = settings.TEST_SUITE,
poll_timeout = settings.POLL_TIMEOUT,
login_page = settings.HOME_NOT_LOGGED_IN,
server_uri = settings.SERVER_URI,
realm_uri = user_profile.realm.uri,
maxfilesize = settings.MAX_FILE_UPLOAD_SIZE,
server_generation = settings.SERVER_GENERATION,
password_auth_enabled = password_auth_enabled(user_profile.realm),
have_initial_messages = user_has_messages,
subbed_info = register_ret['subscriptions'],
unsubbed_info = register_ret['unsubscribed'],
neversubbed_info = register_ret['never_subscribed'],
email_dict = register_ret['email_dict'],
people_list = register_ret['realm_users'],
bot_list = register_ret['realm_bots'],
initial_pointer = register_ret['pointer'],
initial_presences = register_ret['presences'],
initial_servertime = time.time(), # Used for calculating relative presence age
fullname = user_profile.full_name,
email = user_profile.email,
domain = user_profile.realm.domain,
domains = list_of_domains_for_realm(user_profile.realm),
realm_name = register_ret['realm_name'],
realm_invite_required = register_ret['realm_invite_required'],
realm_invite_by_admins_only = register_ret['realm_invite_by_admins_only'],
realm_create_stream_by_admins_only = register_ret['realm_create_stream_by_admins_only'],
realm_allow_message_editing = register_ret['realm_allow_message_editing'],
realm_message_content_edit_limit_seconds = register_ret['realm_message_content_edit_limit_seconds'],
realm_restricted_to_domain = register_ret['realm_restricted_to_domain'],
realm_default_language = register_ret['realm_default_language'],
enter_sends = user_profile.enter_sends,
user_id = user_profile.id,
left_side_userlist = register_ret['left_side_userlist'],
default_language = register_ret['default_language'],
default_language_name = get_language_name(register_ret['default_language']),
language_list_dbl_col = get_language_list_for_templates(register_ret['default_language']),
language_list = get_language_list(),
referrals = register_ret['referrals'],
realm_emoji = register_ret['realm_emoji'],
needs_tutorial = needs_tutorial,
first_in_realm = first_in_realm,
prompt_for_invites = prompt_for_invites,
notifications_stream = notifications_stream,
cross_realm_user_emails = list(get_cross_realm_users()),
# Stream message notification settings:
stream_desktop_notifications_enabled =
user_profile.enable_stream_desktop_notifications,
stream_sounds_enabled = user_profile.enable_stream_sounds,
# Private message and @-mention notification settings:
desktop_notifications_enabled = desktop_notifications_enabled,
sounds_enabled =
user_profile.enable_sounds,
enable_offline_email_notifications =
user_profile.enable_offline_email_notifications,
enable_offline_push_notifications =
user_profile.enable_offline_push_notifications,
enable_online_push_notifications =
user_profile.enable_online_push_notifications,
twenty_four_hour_time = register_ret['twenty_four_hour_time'],
enable_digest_emails = user_profile.enable_digest_emails,
event_queue_id = register_ret['queue_id'],
last_event_id = register_ret['last_event_id'],
max_message_id = register_ret['max_message_id'],
unread_count = approximate_unread_count(user_profile),
furthest_read_time = sent_time_in_epoch_seconds(latest_read),
save_stacktraces = settings.SAVE_FRONTEND_STACKTRACES,
alert_words = register_ret['alert_words'],
muted_topics = register_ret['muted_topics'],
realm_filters = register_ret['realm_filters'],
realm_default_streams = register_ret['realm_default_streams'],
is_admin = user_profile.is_realm_admin,
can_create_streams = user_profile.can_create_streams(),
name_changes_disabled = name_changes_disabled(user_profile.realm),
has_mobile_devices = num_push_devices_for_user(user_profile) > 0,
autoscroll_forever = user_profile.autoscroll_forever,
default_desktop_notifications = user_profile.default_desktop_notifications,
avatar_url = avatar_url(user_profile),
avatar_url_medium = avatar_url(user_profile, medium=True),
mandatory_topics = user_profile.realm.mandatory_topics,
show_digest_email = user_profile.realm.show_digest_email,
presence_disabled = user_profile.realm.presence_disabled,
is_zephyr_mirror_realm = user_profile.realm.is_zephyr_mirror_realm,
)
if narrow_stream is not None:
# In narrow_stream context, initial pointer is just latest message
recipient = get_recipient(Recipient.STREAM, narrow_stream.id)
try:
initial_pointer = Message.objects.filter(recipient=recipient).order_by('id').reverse()[0].id
except IndexError:
initial_pointer = -1
page_params["narrow_stream"] = narrow_stream.name
if narrow_topic is not None:
page_params["narrow_topic"] = narrow_topic
page_params["narrow"] = [dict(operator=term[0], operand=term[1]) for term in narrow]
page_params["max_message_id"] = initial_pointer
page_params["initial_pointer"] = initial_pointer
page_params["have_initial_messages"] = (initial_pointer != -1)
statsd.incr('views.home')
show_invites = True
# Some realms only allow admins to invite users
if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin:
show_invites = False
product_name = "Zulip"
page_params['product_name'] = product_name
request._log_data['extra'] = "[%s]" % (register_ret["queue_id"],)
response = render_to_response('zerver/index.html',
{'user_profile': user_profile,
'page_params' : simplejson.encoder.JSONEncoderForHTML().encode(page_params),
'nofontface': is_buggy_ua(request.META.get("HTTP_USER_AGENT", "Unspecified")),
'avatar_url': avatar_url(user_profile),
'show_debug':
settings.DEBUG and ('show_debug' in request.GET),
'pipeline': settings.PIPELINE_ENABLED,
'show_invites': show_invites,
'is_admin': user_profile.is_realm_admin,
'show_webathena': user_profile.realm.webathena_enabled,
'enable_feedback': settings.ENABLE_FEEDBACK,
'embedded': narrow_stream is not None,
'product_name': product_name
},
request=request)
patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True)
return response
def home_real(request: HttpRequest) -> HttpResponse:
# Before we do any real work, check if the app is banned.
client_user_agent = request.META.get("HTTP_USER_AGENT", "")
(insecure_desktop_app, banned_desktop_app,
auto_update_broken) = is_outdated_desktop_app(client_user_agent)
if banned_desktop_app:
return render(
request,
'zerver/insecure_desktop_app.html',
context={
"auto_update_broken": auto_update_broken,
},
)
(unsupported_browser,
browser_name) = is_unsupported_browser(client_user_agent)
if unsupported_browser:
return render(
request,
'zerver/unsupported_browser.html',
context={
"browser_name": browser_name,
},
)
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
if request.user.is_authenticated:
user_profile = request.user
else: # nocoverage
# This code path should not be reachable because of zulip_login_required above.
user_profile = None
update_last_reminder(user_profile)
statsd.incr('views.home')
# If a user hasn't signed the current Terms of Service, send them there
if need_accept_tos(user_profile):
return accounts_accept_terms(request)
narrow, narrow_stream, narrow_topic = detect_narrowed_window(
request, user_profile)
if user_profile is not None:
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = (first_in_realm
and not PreregistrationUser.objects.filter(
referred_by=user_profile).count())
needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING
else: # nocoverage
first_in_realm = False
prompt_for_invites = False
# The current tutorial doesn't super make sense for logged-out users.
needs_tutorial = False
has_mobile_devices = user_profile is not None and num_push_devices_for_user(
user_profile) > 0
queue_id, page_params = build_page_params_for_home_page_load(
request=request,
user_profile=user_profile,
insecure_desktop_app=insecure_desktop_app,
has_mobile_devices=has_mobile_devices,
narrow=narrow,
narrow_stream=narrow_stream,
narrow_topic=narrow_topic,
first_in_realm=first_in_realm,
prompt_for_invites=prompt_for_invites,
needs_tutorial=needs_tutorial,
)
show_invites, show_add_streams = compute_show_invites_and_add_streams(
user_profile)
billing_info = get_billing_info(user_profile)
request._log_data['extra'] = "[{}]".format(queue_id)
csp_nonce = secrets.token_hex(24)
user_permission_info = get_user_permission_info(user_profile)
navbar_logo_url = compute_navbar_logo_url(page_params)
response = render(request,
'zerver/app/index.html',
context={
'user_profile':
user_profile,
'page_params':
page_params,
'csp_nonce':
csp_nonce,
'search_pills_enabled':
settings.SEARCH_PILLS_ENABLED,
'show_invites':
show_invites,
'show_add_streams':
show_add_streams,
'show_billing':
billing_info.show_billing,
'corporate_enabled':
settings.CORPORATE_ENABLED,
'show_plans':
billing_info.show_plans,
'is_owner':
user_permission_info.is_realm_owner,
'is_admin':
user_permission_info.is_realm_admin,
'is_guest':
user_permission_info.is_guest,
'color_scheme':
user_permission_info.color_scheme,
'navbar_logo_url':
navbar_logo_url,
'show_webathena':
user_permission_info.show_webathena,
'embedded':
narrow_stream is not None,
'invite_as':
PreregistrationUser.INVITE_AS,
'max_file_upload_size_mib':
settings.MAX_FILE_UPLOAD_SIZE,
})
patch_cache_control(response,
no_cache=True,
no_store=True,
must_revalidate=True)
return response
def write_log_line(log_data, path, method, remote_ip, email, client_name,
status_code=200, error_content=None, error_content_iter=None):
# type: (MutableMapping[str, Any], Text, str, str, Text, Text, int, Optional[AnyStr], Optional[Iterable[AnyStr]]) -> None
assert error_content is None or error_content_iter is None
if error_content is not None:
error_content_iter = (error_content,)
# For statsd timer name
if path == '/':
statsd_path = u'webreq'
else:
statsd_path = u"webreq.%s" % (path[1:].replace('/', '.'),)
# Remove non-ascii chars from path (there should be none, if there are it's
# because someone manually entered a nonexistant path), as UTF-8 chars make
# statsd sad when it sends the key name over the socket
statsd_path = statsd_path.encode('ascii', errors='ignore').decode("ascii")
blacklisted_requests = ['do_confirm', 'send_confirm',
'eventslast_event_id', 'webreq.content', 'avatar', 'user_uploads',
'password.reset', 'static', 'json.bots', 'json.users', 'json.streams',
'accounts.unsubscribe', 'apple-touch-icon', 'emoji', 'json.bots',
'upload_file', 'realm_activity', 'user_activity']
suppress_statsd = any((blacklisted in statsd_path for blacklisted in blacklisted_requests))
time_delta = -1
# A time duration of -1 means the StartLogRequests middleware
# didn't run for some reason
optional_orig_delta = ""
if 'time_started' in log_data:
time_delta = time.time() - log_data['time_started']
if 'time_stopped' in log_data:
orig_time_delta = time_delta
time_delta = ((log_data['time_stopped'] - log_data['time_started']) +
(time.time() - log_data['time_restarted']))
optional_orig_delta = " (lp: %s)" % (format_timedelta(orig_time_delta),)
remote_cache_output = ""
if 'remote_cache_time_start' in log_data:
remote_cache_time_delta = get_remote_cache_time() - log_data['remote_cache_time_start']
remote_cache_count_delta = get_remote_cache_requests() - log_data['remote_cache_requests_start']
if 'remote_cache_requests_stopped' in log_data:
# (now - restarted) + (stopped - start) = (now - start) + (stopped - restarted)
remote_cache_time_delta += (log_data['remote_cache_time_stopped'] -
log_data['remote_cache_time_restarted'])
remote_cache_count_delta += (log_data['remote_cache_requests_stopped'] -
log_data['remote_cache_requests_restarted'])
if (remote_cache_time_delta > 0.005):
remote_cache_output = " (mem: %s/%s)" % (format_timedelta(remote_cache_time_delta),
remote_cache_count_delta)
if not suppress_statsd:
statsd.timing("%s.remote_cache.time" % (statsd_path,), timedelta_ms(remote_cache_time_delta))
statsd.incr("%s.remote_cache.querycount" % (statsd_path,), remote_cache_count_delta)
startup_output = ""
if 'startup_time_delta' in log_data and log_data["startup_time_delta"] > 0.005:
startup_output = " (+start: %s)" % (format_timedelta(log_data["startup_time_delta"]))
bugdown_output = ""
if 'bugdown_time_start' in log_data:
bugdown_time_delta = get_bugdown_time() - log_data['bugdown_time_start']
bugdown_count_delta = get_bugdown_requests() - log_data['bugdown_requests_start']
if 'bugdown_requests_stopped' in log_data:
# (now - restarted) + (stopped - start) = (now - start) + (stopped - restarted)
bugdown_time_delta += (log_data['bugdown_time_stopped'] -
log_data['bugdown_time_restarted'])
bugdown_count_delta += (log_data['bugdown_requests_stopped'] -
log_data['bugdown_requests_restarted'])
if (bugdown_time_delta > 0.005):
bugdown_output = " (md: %s/%s)" % (format_timedelta(bugdown_time_delta),
bugdown_count_delta)
if not suppress_statsd:
statsd.timing("%s.markdown.time" % (statsd_path,), timedelta_ms(bugdown_time_delta))
statsd.incr("%s.markdown.count" % (statsd_path,), bugdown_count_delta)
# Get the amount of time spent doing database queries
db_time_output = ""
queries = connection.connection.queries if connection.connection is not None else []
if len(queries) > 0:
query_time = sum(float(query.get('time', 0)) for query in queries)
db_time_output = " (db: %s/%sq)" % (format_timedelta(query_time),
len(queries))
if not suppress_statsd:
# Log ms, db ms, and num queries to statsd
statsd.timing("%s.dbtime" % (statsd_path,), timedelta_ms(query_time))
statsd.incr("%s.dbq" % (statsd_path,), len(queries))
statsd.timing("%s.total" % (statsd_path,), timedelta_ms(time_delta))
if 'extra' in log_data:
extra_request_data = " %s" % (log_data['extra'],)
else:
extra_request_data = ""
logger_client = "(%s via %s)" % (email, client_name)
logger_timing = ('%5s%s%s%s%s%s %s' %
(format_timedelta(time_delta), optional_orig_delta,
remote_cache_output, bugdown_output,
db_time_output, startup_output, path))
logger_line = ('%-15s %-7s %3d %s%s %s' %
(remote_ip, method, status_code,
logger_timing, extra_request_data, logger_client))
if (status_code in [200, 304] and method == "GET" and path.startswith("/static")):
logger.debug(logger_line)
else:
logger.info(logger_line)
if (is_slow_query(time_delta, path)):
queue_json_publish("slow_queries", "%s (%s)" % (logger_line, email), lambda e: None)
if settings.PROFILE_ALL_REQUESTS:
log_data["prof"].disable()
profile_path = "/tmp/profile.data.%s.%s" % (path.split("/")[-1], int(time_delta * 1000),)
log_data["prof"].dump_stats(profile_path)
# Log some additional data whenever we return certain 40x errors
if 400 <= status_code < 500 and status_code not in [401, 404, 405]:
assert error_content_iter is not None
error_content_list = list(error_content_iter)
if error_content_list:
error_data = u''
elif isinstance(error_content_list[0], Text):
error_data = u''.join(error_content_list)
elif isinstance(error_content_list[0], binary_type):
error_data = repr(b''.join(error_content_list))
if len(error_data) > 100:
error_data = u"[content more than 100 characters]"
logger.info('status=%3d, data=%s, uid=%s' % (status_code, error_data, email))
def home_real(request: HttpRequest) -> HttpResponse:
# Before we do any real work, check if the app is banned.
client_user_agent = request.META.get("HTTP_USER_AGENT", "")
(insecure_desktop_app, banned_desktop_app, auto_update_broken) = is_outdated_desktop_app(
client_user_agent)
if banned_desktop_app:
return render(
request,
'zerver/insecure_desktop_app.html',
context={
"auto_update_broken": auto_update_broken,
},
)
(unsupported_browser, browser_name) = is_unsupported_browser(client_user_agent)
if unsupported_browser:
return render(
request,
'zerver/unsupported_browser.html',
context={
"browser_name": browser_name,
},
)
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
if request.user.is_authenticated:
user_profile = request.user
else: # nocoverage
# This code path should not be reachable because of zulip_login_required above.
user_profile = None
# If a user hasn't signed the current Terms of Service, send them there
if need_accept_tos(user_profile):
return accounts_accept_terms(request)
narrow, narrow_stream, narrow_topic = detect_narrowed_window(request, user_profile)
client_capabilities = {
'notification_settings_null': True,
'bulk_message_deletion': True,
'user_avatar_url_field_optional': True,
}
register_ret = do_events_register(user_profile, request.client,
apply_markdown=True, client_gravatar=True,
slim_presence=True,
client_capabilities=client_capabilities,
narrow=narrow)
update_last_reminder(user_profile)
if user_profile is not None:
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = (
first_in_realm and
not PreregistrationUser.objects.filter(referred_by=user_profile).count()
)
needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING
else: # nocoverage
first_in_realm = False
prompt_for_invites = False
# The current tutorial doesn't super make sense for logged-out users.
needs_tutorial = False
furthest_read_time = get_furthest_read_time(user_profile)
# We pick a language for the user as follows:
# * First priority is the language in the URL, for debugging.
# * If not in the URL, we use the language from the user's settings.
request_language = translation.get_language_from_path(request.path_info)
if request_language is None:
request_language = register_ret['default_language']
translation.activate(request_language)
# We also save the language to the user's session, so that
# something reasonable will happen in logged-in portico pages.
request.session[translation.LANGUAGE_SESSION_KEY] = translation.get_language()
two_fa_enabled = settings.TWO_FACTOR_AUTHENTICATION_ENABLED and user_profile is not None
# Pass parameters to the client-side JavaScript code.
# These end up in a global JavaScript Object named 'page_params'.
page_params = dict(
# Server settings.
debug_mode = settings.DEBUG,
test_suite = settings.TEST_SUITE,
poll_timeout = settings.POLL_TIMEOUT,
insecure_desktop_app = insecure_desktop_app,
login_page = settings.HOME_NOT_LOGGED_IN,
root_domain_uri = settings.ROOT_DOMAIN_URI,
save_stacktraces = settings.SAVE_FRONTEND_STACKTRACES,
warn_no_email = settings.WARN_NO_EMAIL,
search_pills_enabled = settings.SEARCH_PILLS_ENABLED,
# Misc. extra data.
initial_servertime = time.time(), # Used for calculating relative presence age
default_language_name = get_language_name(register_ret['default_language']),
language_list_dbl_col = get_language_list_for_templates(register_ret['default_language']),
language_list = get_language_list(),
needs_tutorial = needs_tutorial,
first_in_realm = first_in_realm,
prompt_for_invites = prompt_for_invites,
furthest_read_time = furthest_read_time,
has_mobile_devices = user_profile is not None and num_push_devices_for_user(user_profile) > 0,
bot_types = get_bot_types(user_profile),
two_fa_enabled = two_fa_enabled,
# Adding two_fa_enabled as condition saves us 3 queries when
# 2FA is not enabled.
two_fa_enabled_user = two_fa_enabled and bool(default_device(user_profile)),
)
undesired_register_ret_fields = [
'streams',
]
for field_name in set(register_ret.keys()) - set(undesired_register_ret_fields):
page_params[field_name] = register_ret[field_name]
if narrow_stream is not None:
# In narrow_stream context, initial pointer is just latest message
recipient = narrow_stream.recipient
try:
max_message_id = Message.objects.filter(recipient=recipient).order_by('id').reverse()[0].id
except IndexError:
max_message_id = -1
page_params["narrow_stream"] = narrow_stream.name
if narrow_topic is not None:
page_params["narrow_topic"] = narrow_topic
page_params["narrow"] = [dict(operator=term[0], operand=term[1]) for term in narrow]
page_params["max_message_id"] = max_message_id
page_params["enable_desktop_notifications"] = False
statsd.incr('views.home')
show_invites, show_add_streams = compute_show_invites_and_add_streams(user_profile)
show_billing = False
show_plans = False
if settings.CORPORATE_ENABLED and user_profile is not None:
from corporate.models import CustomerPlan, get_customer_by_realm
if user_profile.has_billing_access:
customer = get_customer_by_realm(user_profile.realm)
if customer is not None:
if customer.sponsorship_pending:
show_billing = True
elif CustomerPlan.objects.filter(customer=customer).exists():
show_billing = True
if user_profile.realm.plan_type == Realm.LIMITED:
show_plans = True
request._log_data['extra'] = "[{}]".format(register_ret["queue_id"])
page_params['translation_data'] = {}
if request_language != 'en':
page_params['translation_data'] = get_language_translation_data(request_language)
csp_nonce = generate_random_token(48)
if user_profile is not None:
color_scheme = user_profile.color_scheme
is_guest = user_profile.is_guest
is_realm_owner = user_profile.is_realm_owner
is_realm_admin = user_profile.is_realm_admin
show_webathena = user_profile.realm.webathena_enabled
else: # nocoverage
color_scheme = UserProfile.COLOR_SCHEME_AUTOMATIC
is_guest = False
is_realm_admin = False
is_realm_owner = False
show_webathena = False
navbar_logo_url = compute_navbar_logo_url(page_params)
# print("----------------->")
# new_realm_users = []
# user_initial = user_profile.full_name[0]
# pattern = '^{}'.format(user_initial)
# for _ in range(len(page_params['realm_users'])):
# current_user_fullname = page_params['realm_users'][_]['full_name']
# initials_matched_user = re.match(pattern,current_user_fullname)
# if initials_matched_user:
# new_realm_users.append(page_params['realm_users'][_])
# page_params['realm_users'] = new_realm_users
# print(page_params['realm_users'])
# print('\n')
# print(page_params.keys())
# print("-----------------> UserProfile objects")
# print(UserProfile.objects.all())
response = render(request, 'zerver/app/index.html',
context={'user_profile': user_profile,
'page_params': page_params,
'csp_nonce': csp_nonce,
'search_pills_enabled': settings.SEARCH_PILLS_ENABLED,
'show_invites': show_invites,
'show_add_streams': show_add_streams,
'show_billing': show_billing,
'corporate_enabled': settings.CORPORATE_ENABLED,
'show_plans': show_plans,
'is_owner': is_realm_owner,
'is_admin': is_realm_admin,
'is_guest': is_guest,
'color_scheme': color_scheme,
'navbar_logo_url': navbar_logo_url,
'show_webathena': show_webathena,
'embedded': narrow_stream is not None,
'invite_as': PreregistrationUser.INVITE_AS,
'max_file_upload_size_mib': settings.MAX_FILE_UPLOAD_SIZE,
})
patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True)
# print("########################### | These are page_params keys | ##############")
# print(page_params.keys())
# print("########################### | Realm Users | #############################")
# print(page_params['realm_users'])
# print("############################ | Presences |############################")
# print("\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/")
# print(page_params['presences'])
return response
def wrapped_func(*args, **kwargs):
ret = func(*args, **kwargs)
statsd.incr(counter, val)
return ret
def home_real(request: HttpRequest) -> HttpResponse:
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
user_profile = request.user
# If a user hasn't signed the current Terms of Service, send them there
if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \
int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version():
return accounts_accept_terms(request)
narrow = [] # type: List[List[str]]
narrow_stream = None
narrow_topic = request.GET.get("topic")
if request.GET.get("stream"):
try:
narrow_stream_name = request.GET.get("stream")
(narrow_stream, ignored_rec,
ignored_sub) = access_stream_by_name(user_profile,
narrow_stream_name)
narrow = [["stream", narrow_stream.name]]
except Exception:
logging.exception("Narrow parsing exception",
extra=dict(request=request))
if narrow_stream is not None and narrow_topic is not None:
narrow.append(["topic", narrow_topic])
register_ret = do_events_register(user_profile,
request.client,
apply_markdown=True,
client_gravatar=True,
narrow=narrow)
user_has_messages = (register_ret['max_message_id'] != -1)
# Reset our don't-spam-users-with-email counter since the
# user has since logged in
if user_profile.last_reminder is not None: # nocoverage
# TODO: Look into the history of last_reminder; we may have
# eliminated that as a useful concept for non-bot users.
user_profile.last_reminder = None
user_profile.save(update_fields=["last_reminder"])
# Brand new users get narrowed to PM with welcome-bot
needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = first_in_realm and \
not PreregistrationUser.objects.filter(referred_by=user_profile).count()
if user_profile.pointer == -1 and user_has_messages:
# Put the new user's pointer at the bottom
#
# This improves performance, because we limit backfilling of messages
# before the pointer. It's also likely that someone joining an
# organization is interested in recent messages more than the very
# first messages on the system.
register_ret['pointer'] = register_ret['max_message_id']
user_profile.last_pointer_updater = request.session.session_key
if user_profile.pointer == -1:
latest_read = None
else:
latest_read = get_usermessage_by_message_id(user_profile,
user_profile.pointer)
if latest_read is None:
# Don't completely fail if your saved pointer ID is invalid
logging.warning("%s has invalid pointer %s" %
(user_profile.email, user_profile.pointer))
# We pick a language for the user as follows:
# * First priority is the language in the URL, for debugging.
# * If not in the URL, we use the language from the user's settings.
request_language = translation.get_language_from_path(request.path_info)
if request_language is None:
request_language = register_ret['default_language']
translation.activate(request_language)
# We also save the language to the user's session, so that
# something reasonable will happen in logged-in portico pages.
request.session[
translation.LANGUAGE_SESSION_KEY] = translation.get_language()
two_fa_enabled = settings.TWO_FACTOR_AUTHENTICATION_ENABLED
# Pass parameters to the client-side JavaScript code.
# These end up in a global JavaScript Object named 'page_params'.
page_params = dict(
# Server settings.
development_environment=settings.DEVELOPMENT,
debug_mode=settings.DEBUG,
test_suite=settings.TEST_SUITE,
poll_timeout=settings.POLL_TIMEOUT,
login_page=settings.HOME_NOT_LOGGED_IN,
root_domain_uri=settings.ROOT_DOMAIN_URI,
maxfilesize=settings.MAX_FILE_UPLOAD_SIZE,
max_avatar_file_size=settings.MAX_AVATAR_FILE_SIZE,
server_generation=settings.SERVER_GENERATION,
use_websockets=settings.USE_WEBSOCKETS,
save_stacktraces=settings.SAVE_FRONTEND_STACKTRACES,
warn_no_email=settings.WARN_NO_EMAIL,
server_inline_image_preview=settings.INLINE_IMAGE_PREVIEW,
server_inline_url_embed_preview=settings.INLINE_URL_EMBED_PREVIEW,
password_min_length=settings.PASSWORD_MIN_LENGTH,
password_min_guesses=settings.PASSWORD_MIN_GUESSES,
jitsi_server_url=settings.JITSI_SERVER_URL,
search_pills_enabled=settings.SEARCH_PILLS_ENABLED,
# Misc. extra data.
have_initial_messages=user_has_messages,
initial_servertime=time.time(
), # Used for calculating relative presence age
default_language_name=get_language_name(
register_ret['default_language']),
language_list_dbl_col=get_language_list_for_templates(
register_ret['default_language']),
language_list=get_language_list(),
needs_tutorial=needs_tutorial,
first_in_realm=first_in_realm,
prompt_for_invites=prompt_for_invites,
furthest_read_time=sent_time_in_epoch_seconds(latest_read),
has_mobile_devices=num_push_devices_for_user(user_profile) > 0,
bot_types=get_bot_types(user_profile),
two_fa_enabled=two_fa_enabled,
# Adding two_fa_enabled as condition saves us 3 queries when
# 2FA is not enabled.
two_fa_enabled_user=two_fa_enabled
and bool(default_device(user_profile)),
)
undesired_register_ret_fields = [
'streams',
]
for field_name in set(
register_ret.keys()) - set(undesired_register_ret_fields):
page_params[field_name] = register_ret[field_name]
if narrow_stream is not None:
# In narrow_stream context, initial pointer is just latest message
recipient = get_stream_recipient(narrow_stream.id)
try:
initial_pointer = Message.objects.filter(
recipient=recipient).order_by('id').reverse()[0].id
except IndexError:
initial_pointer = -1
page_params["narrow_stream"] = narrow_stream.name
if narrow_topic is not None:
page_params["narrow_topic"] = narrow_topic
page_params["narrow"] = [
dict(operator=term[0], operand=term[1]) for term in narrow
]
page_params["max_message_id"] = initial_pointer
page_params["pointer"] = initial_pointer
page_params["have_initial_messages"] = (initial_pointer != -1)
page_params["enable_desktop_notifications"] = False
statsd.incr('views.home')
show_invites = True
# Some realms only allow admins to invite users
if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin:
show_invites = False
if user_profile.is_guest:
show_invites = False
show_billing = False
show_plans = False
if settings.CORPORATE_ENABLED:
from corporate.models import Customer
if user_profile.is_billing_admin or user_profile.is_realm_admin:
customer = Customer.objects.filter(
realm=user_profile.realm).first()
if customer is not None and customer.has_billing_relationship:
show_billing = True
if user_profile.realm.plan_type == Realm.LIMITED:
show_plans = True
request._log_data['extra'] = "[%s]" % (register_ret["queue_id"], )
page_params['translation_data'] = {}
if request_language != 'en':
page_params['translation_data'] = get_language_translation_data(
request_language)
csp_nonce = generate_random_token(48)
emojiset = user_profile.emojiset
if emojiset == UserProfile.TEXT_EMOJISET:
# If current emojiset is `TEXT_EMOJISET`, then fallback to
# GOOGLE_EMOJISET for picking which spritesheet's CSS to
# include (and thus how to display emojis in the emoji picker
# and composebox typeahead).
emojiset = UserProfile.GOOGLE_BLOB_EMOJISET
response = render(
request,
'zerver/app/index.html',
context={
'user_profile': user_profile,
'emojiset': emojiset,
'page_params': JSONEncoderForHTML().encode(page_params),
'csp_nonce': csp_nonce,
'avatar_url': avatar_url(user_profile),
'show_debug': settings.DEBUG and ('show_debug' in request.GET),
'pipeline': settings.PIPELINE_ENABLED,
'search_pills_enabled': settings.SEARCH_PILLS_ENABLED,
'show_invites': show_invites,
'show_billing': show_billing,
'show_plans': show_plans,
'is_admin': user_profile.is_realm_admin,
'is_guest': user_profile.is_guest,
'show_webathena': user_profile.realm.webathena_enabled,
'enable_feedback': settings.ENABLE_FEEDBACK,
'embedded': narrow_stream is not None,
},
)
patch_cache_control(response,
no_cache=True,
no_store=True,
must_revalidate=True)
return response
def home_real(request):
# type: (HttpRequest) -> HttpResponse
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
user_profile = request.user
# If a user hasn't signed the current Terms of Service, send them there
if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \
int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version():
return accounts_accept_terms(request)
narrow = [] # type: List[List[Text]]
narrow_stream = None
narrow_topic = request.GET.get("topic")
if request.GET.get("stream"):
try:
narrow_stream_name = request.GET.get("stream")
(narrow_stream, ignored_rec, ignored_sub) = access_stream_by_name(
user_profile, narrow_stream_name)
narrow = [["stream", narrow_stream.name]]
except Exception:
logging.exception("Narrow parsing")
if narrow_stream is not None and narrow_topic is not None:
narrow.append(["topic", narrow_topic])
register_ret = do_events_register(user_profile, request.client,
apply_markdown=True, narrow=narrow)
user_has_messages = (register_ret['max_message_id'] != -1)
# Reset our don't-spam-users-with-email counter since the
# user has since logged in
if user_profile.last_reminder is not None:
user_profile.last_reminder = None
user_profile.save(update_fields=["last_reminder"])
# Brand new users get the tutorial
needs_tutorial = settings.TUTORIAL_ENABLED and \
user_profile.tutorial_status != UserProfile.TUTORIAL_FINISHED
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = first_in_realm and \
not PreregistrationUser.objects.filter(referred_by=user_profile).count()
if user_profile.pointer == -1 and user_has_messages:
# Put the new user's pointer at the bottom
#
# This improves performance, because we limit backfilling of messages
# before the pointer. It's also likely that someone joining an
# organization is interested in recent messages more than the very
# first messages on the system.
register_ret['pointer'] = register_ret['max_message_id']
user_profile.last_pointer_updater = request.session.session_key
if user_profile.pointer == -1:
latest_read = None
else:
try:
latest_read = UserMessage.objects.get(user_profile=user_profile,
message__id=user_profile.pointer)
except UserMessage.DoesNotExist:
# Don't completely fail if your saved pointer ID is invalid
logging.warning("%s has invalid pointer %s" % (user_profile.email, user_profile.pointer))
latest_read = None
# Set default language and make it persist
default_language = register_ret['default_language']
url_lang = '/{}'.format(request.LANGUAGE_CODE)
if not request.path.startswith(url_lang):
translation.activate(default_language)
request.session[translation.LANGUAGE_SESSION_KEY] = default_language
# Pass parameters to the client-side JavaScript code.
# These end up in a global JavaScript Object named 'page_params'.
page_params = dict(
# Server settings.
share_the_love = settings.SHARE_THE_LOVE,
development_environment = settings.DEVELOPMENT,
debug_mode = settings.DEBUG,
test_suite = settings.TEST_SUITE,
poll_timeout = settings.POLL_TIMEOUT,
login_page = settings.HOME_NOT_LOGGED_IN,
server_uri = settings.SERVER_URI,
maxfilesize = settings.MAX_FILE_UPLOAD_SIZE,
max_avatar_file_size = settings.MAX_AVATAR_FILE_SIZE,
server_generation = settings.SERVER_GENERATION,
use_websockets = settings.USE_WEBSOCKETS,
save_stacktraces = settings.SAVE_FRONTEND_STACKTRACES,
server_inline_image_preview = settings.INLINE_IMAGE_PREVIEW,
server_inline_url_embed_preview = settings.INLINE_URL_EMBED_PREVIEW,
# Misc. extra data.
have_initial_messages = user_has_messages,
initial_servertime = time.time(), # Used for calculating relative presence age
default_language_name = get_language_name(register_ret['default_language']),
language_list_dbl_col = get_language_list_for_templates(register_ret['default_language']),
language_list = get_language_list(),
needs_tutorial = needs_tutorial,
first_in_realm = first_in_realm,
prompt_for_invites = prompt_for_invites,
unread_count = approximate_unread_count(user_profile),
furthest_read_time = sent_time_in_epoch_seconds(latest_read),
has_mobile_devices = num_push_devices_for_user(user_profile) > 0,
)
undesired_register_ret_fields = [
'streams',
]
for field_name in set(register_ret.keys()) - set(undesired_register_ret_fields):
page_params[field_name] = register_ret[field_name]
if narrow_stream is not None:
# In narrow_stream context, initial pointer is just latest message
recipient = get_recipient(Recipient.STREAM, narrow_stream.id)
try:
initial_pointer = Message.objects.filter(recipient=recipient).order_by('id').reverse()[0].id
except IndexError:
initial_pointer = -1
page_params["narrow_stream"] = narrow_stream.name
if narrow_topic is not None:
page_params["narrow_topic"] = narrow_topic
page_params["narrow"] = [dict(operator=term[0], operand=term[1]) for term in narrow]
page_params["max_message_id"] = initial_pointer
page_params["pointer"] = initial_pointer
page_params["have_initial_messages"] = (initial_pointer != -1)
page_params["enable_desktop_notifications"] = False
statsd.incr('views.home')
show_invites = True
# Some realms only allow admins to invite users
if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin:
show_invites = False
request._log_data['extra'] = "[%s]" % (register_ret["queue_id"],)
response = render(request, 'zerver/index.html',
context={'user_profile': user_profile,
'page_params': simplejson.encoder.JSONEncoderForHTML().encode(page_params),
'nofontface': is_buggy_ua(request.META.get("HTTP_USER_AGENT", "Unspecified")),
'avatar_url': avatar_url(user_profile),
'show_debug':
settings.DEBUG and ('show_debug' in request.GET),
'pipeline': settings.PIPELINE_ENABLED,
'show_invites': show_invites,
'is_admin': user_profile.is_realm_admin,
'show_webathena': user_profile.realm.webathena_enabled,
'enable_feedback': settings.ENABLE_FEEDBACK,
'embedded': narrow_stream is not None,
},)
patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True)
return response
def home_real(request: HttpRequest) -> HttpResponse:
# Before we do any real work, check if the app is banned.
client_user_agent = request.headers.get("User-Agent", "")
(insecure_desktop_app, banned_desktop_app,
auto_update_broken) = is_outdated_desktop_app(client_user_agent)
if banned_desktop_app:
return render(
request,
"zerver/insecure_desktop_app.html",
context={
"auto_update_broken": auto_update_broken,
},
)
(unsupported_browser,
browser_name) = is_unsupported_browser(client_user_agent)
if unsupported_browser:
return render(
request,
"zerver/unsupported_browser.html",
context={
"browser_name": browser_name,
},
)
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
if request.user.is_authenticated:
user_profile = request.user
realm = user_profile.realm
else:
realm = get_valid_realm_from_request(request)
# We load the spectator experience. We fall through to the shared code
# for loading the application, with user_profile=None encoding
# that we're a spectator, not a logged-in user.
user_profile = None
update_last_reminder(user_profile)
statsd.incr("views.home")
# If a user hasn't signed the current Terms of Service, send them there
if need_accept_tos(user_profile):
return accounts_accept_terms(request)
narrow, narrow_stream, narrow_topic = detect_narrowed_window(
request, user_profile)
if user_profile is not None:
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = (first_in_realm
and not PreregistrationUser.objects.filter(
referred_by=user_profile).count())
needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING
else:
first_in_realm = False
prompt_for_invites = False
# The current tutorial doesn't super make sense for logged-out users.
needs_tutorial = False
queue_id, page_params = build_page_params_for_home_page_load(
request=request,
user_profile=user_profile,
realm=realm,
insecure_desktop_app=insecure_desktop_app,
narrow=narrow,
narrow_stream=narrow_stream,
narrow_topic=narrow_topic,
first_in_realm=first_in_realm,
prompt_for_invites=prompt_for_invites,
needs_tutorial=needs_tutorial,
)
log_data = RequestNotes.get_notes(request).log_data
assert log_data is not None
log_data["extra"] = f"[{queue_id}]"
csp_nonce = secrets.token_hex(24)
user_permission_info = get_user_permission_info(user_profile)
response = render(
request,
"zerver/app/index.html",
context={
"user_profile": user_profile,
"page_params": page_params,
"csp_nonce": csp_nonce,
"color_scheme": user_permission_info.color_scheme,
},
)
patch_cache_control(response,
no_cache=True,
no_store=True,
must_revalidate=True)
return response
def wrapped_func(*args, **kwargs):
# type: (*Any, **Any) -> Any
ret = func(*args, **kwargs)
statsd.incr(counter, val)
return ret
def wrapped_func(*args: ParamT.args,
**kwargs: ParamT.kwargs) -> ReturnT:
ret = func(*args, **kwargs)
statsd.incr(counter, val)
return ret
def home_real(request: HttpRequest) -> HttpResponse:
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
user_profile = request.user
# If a user hasn't signed the current Terms of Service, send them there
if settings.TERMS_OF_SERVICE is not None and settings.TOS_VERSION is not None and \
int(settings.TOS_VERSION.split('.')[0]) > user_profile.major_tos_version():
return accounts_accept_terms(request)
narrow = [] # type: List[List[str]]
narrow_stream = None
narrow_topic = request.GET.get("topic")
if request.GET.get("stream"):
try:
narrow_stream_name = request.GET.get("stream")
(narrow_stream, ignored_rec, ignored_sub) = access_stream_by_name(
user_profile, narrow_stream_name)
narrow = [["stream", narrow_stream.name]]
except Exception:
logging.exception("Narrow parsing exception", extra=dict(request=request))
if narrow_stream is not None and narrow_topic is not None:
narrow.append(["topic", narrow_topic])
register_ret = do_events_register(user_profile, request.client,
apply_markdown=True, client_gravatar=True,
narrow=narrow)
user_has_messages = (register_ret['max_message_id'] != -1)
# Reset our don't-spam-users-with-email counter since the
# user has since logged in
if user_profile.last_reminder is not None: # nocoverage
# TODO: Look into the history of last_reminder; we may have
# eliminated that as a useful concept for non-bot users.
user_profile.last_reminder = None
user_profile.save(update_fields=["last_reminder"])
# Brand new users get narrowed to PM with welcome-bot
needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = first_in_realm and \
not PreregistrationUser.objects.filter(referred_by=user_profile).count()
if user_profile.pointer == -1 and user_has_messages:
# Put the new user's pointer at the bottom
#
# This improves performance, because we limit backfilling of messages
# before the pointer. It's also likely that someone joining an
# organization is interested in recent messages more than the very
# first messages on the system.
register_ret['pointer'] = register_ret['max_message_id']
user_profile.last_pointer_updater = request.session.session_key
if user_profile.pointer == -1:
latest_read = None
else:
latest_read = get_usermessage_by_message_id(user_profile, user_profile.pointer)
if latest_read is None:
# Don't completely fail if your saved pointer ID is invalid
logging.warning("%s has invalid pointer %s" % (user_profile.email, user_profile.pointer))
# We pick a language for the user as follows:
# * First priority is the language in the URL, for debugging.
# * If not in the URL, we use the language from the user's settings.
request_language = translation.get_language_from_path(request.path_info)
if request_language is None:
request_language = register_ret['default_language']
translation.activate(request_language)
# We also save the language to the user's session, so that
# something reasonable will happen in logged-in portico pages.
request.session[translation.LANGUAGE_SESSION_KEY] = translation.get_language()
two_fa_enabled = settings.TWO_FACTOR_AUTHENTICATION_ENABLED
# Pass parameters to the client-side JavaScript code.
# These end up in a global JavaScript Object named 'page_params'.
page_params = dict(
# Server settings.
development_environment = settings.DEVELOPMENT,
debug_mode = settings.DEBUG,
test_suite = settings.TEST_SUITE,
poll_timeout = settings.POLL_TIMEOUT,
login_page = settings.HOME_NOT_LOGGED_IN,
root_domain_uri = settings.ROOT_DOMAIN_URI,
maxfilesize = settings.MAX_FILE_UPLOAD_SIZE,
max_avatar_file_size = settings.MAX_AVATAR_FILE_SIZE,
server_generation = settings.SERVER_GENERATION,
use_websockets = settings.USE_WEBSOCKETS,
save_stacktraces = settings.SAVE_FRONTEND_STACKTRACES,
warn_no_email = settings.WARN_NO_EMAIL,
server_inline_image_preview = settings.INLINE_IMAGE_PREVIEW,
server_inline_url_embed_preview = settings.INLINE_URL_EMBED_PREVIEW,
password_min_length = settings.PASSWORD_MIN_LENGTH,
password_min_guesses = settings.PASSWORD_MIN_GUESSES,
jitsi_server_url = settings.JITSI_SERVER_URL,
search_pills_enabled = settings.SEARCH_PILLS_ENABLED,
# Misc. extra data.
have_initial_messages = user_has_messages,
initial_servertime = time.time(), # Used for calculating relative presence age
default_language_name = get_language_name(register_ret['default_language']),
language_list_dbl_col = get_language_list_for_templates(register_ret['default_language']),
language_list = get_language_list(),
needs_tutorial = needs_tutorial,
first_in_realm = first_in_realm,
prompt_for_invites = prompt_for_invites,
furthest_read_time = sent_time_in_epoch_seconds(latest_read),
has_mobile_devices = num_push_devices_for_user(user_profile) > 0,
bot_types = get_bot_types(user_profile),
two_fa_enabled = two_fa_enabled,
# Adding two_fa_enabled as condition saves us 3 queries when
# 2FA is not enabled.
two_fa_enabled_user = two_fa_enabled and bool(default_device(user_profile)),
)
undesired_register_ret_fields = [
'streams',
]
for field_name in set(register_ret.keys()) - set(undesired_register_ret_fields):
page_params[field_name] = register_ret[field_name]
if narrow_stream is not None:
# In narrow_stream context, initial pointer is just latest message
recipient = get_stream_recipient(narrow_stream.id)
try:
initial_pointer = Message.objects.filter(recipient=recipient).order_by('id').reverse()[0].id
except IndexError:
initial_pointer = -1
page_params["narrow_stream"] = narrow_stream.name
if narrow_topic is not None:
page_params["narrow_topic"] = narrow_topic
page_params["narrow"] = [dict(operator=term[0], operand=term[1]) for term in narrow]
page_params["max_message_id"] = initial_pointer
page_params["pointer"] = initial_pointer
page_params["have_initial_messages"] = (initial_pointer != -1)
page_params["enable_desktop_notifications"] = False
statsd.incr('views.home')
show_invites = True
# Some realms only allow admins to invite users
if user_profile.realm.invite_by_admins_only and not user_profile.is_realm_admin:
show_invites = False
if user_profile.is_guest:
show_invites = False
show_billing = False
show_plans = False
if settings.CORPORATE_ENABLED:
from corporate.models import Customer
if user_profile.is_billing_admin or user_profile.is_realm_admin:
customer = Customer.objects.filter(realm=user_profile.realm).first()
if customer is not None and customer.has_billing_relationship:
show_billing = True
if user_profile.realm.plan_type == Realm.LIMITED:
show_plans = True
request._log_data['extra'] = "[%s]" % (register_ret["queue_id"],)
page_params['translation_data'] = {}
if request_language != 'en':
page_params['translation_data'] = get_language_translation_data(request_language)
csp_nonce = generate_random_token(48)
emojiset = user_profile.emojiset
if emojiset == UserProfile.TEXT_EMOJISET:
# If current emojiset is `TEXT_EMOJISET`, then fallback to
# GOOGLE_EMOJISET for picking which spritesheet's CSS to
# include (and thus how to display emojis in the emoji picker
# and composebox typeahead).
emojiset = UserProfile.GOOGLE_BLOB_EMOJISET
response = render(request, 'zerver/app/index.html',
context={'user_profile': user_profile,
'emojiset': emojiset,
'page_params': JSONEncoderForHTML().encode(page_params),
'csp_nonce': csp_nonce,
'avatar_url': avatar_url(user_profile),
'show_debug':
settings.DEBUG and ('show_debug' in request.GET),
'pipeline': settings.PIPELINE_ENABLED,
'search_pills_enabled': settings.SEARCH_PILLS_ENABLED,
'show_invites': show_invites,
'show_billing': show_billing,
'show_plans': show_plans,
'is_admin': user_profile.is_realm_admin,
'is_guest': user_profile.is_guest,
'show_webathena': user_profile.realm.webathena_enabled,
'enable_feedback': settings.ENABLE_FEEDBACK,
'embedded': narrow_stream is not None,
'invite_as': PreregistrationUser.INVITE_AS,
},)
patch_cache_control(response, no_cache=True, no_store=True, must_revalidate=True)
return response
def home_real(request: HttpRequest) -> HttpResponse:
# Before we do any real work, check if the app is banned.
client_user_agent = request.META.get("HTTP_USER_AGENT", "")
(insecure_desktop_app, banned_desktop_app,
auto_update_broken) = is_outdated_desktop_app(client_user_agent)
if banned_desktop_app:
return render(
request,
"zerver/insecure_desktop_app.html",
context={
"auto_update_broken": auto_update_broken,
},
)
(unsupported_browser,
browser_name) = is_unsupported_browser(client_user_agent)
if unsupported_browser:
return render(
request,
"zerver/unsupported_browser.html",
context={
"browser_name": browser_name,
},
)
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
if request.user.is_authenticated:
user_profile = request.user
realm = user_profile.realm
# User is logged in and hence no longer `prefers_web_public_view`.
if "prefers_web_public_view" in request.session.keys():
del request.session["prefers_web_public_view"]
else:
realm = get_valid_realm_from_request(request)
# TODO: Ideally, we'd open Zulip directly as a spectator if
# the URL had clicked a link to content on a web-public
# stream. We could maybe do this by parsing `next`, but it's
# not super convenient with Zulip's hash-based URL scheme.
# The "Access without an account" button on the login page
# submits a POST to this page with this hidden field set.
if request.POST.get("prefers_web_public_view") == "true":
request.session["prefers_web_public_view"] = True
# We serve a redirect here, rather than serving a page, to
# avoid browser "Confirm form resubmission" prompts on reload.
redirect_to = get_safe_redirect_to(request.POST.get("next"),
realm.uri)
return redirect(redirect_to)
prefers_web_public_view = request.session.get(
"prefers_web_public_view")
if not prefers_web_public_view:
# For users who haven't opted into the spectator
# experience, we redirect to the login page.
return zulip_redirect_to_login(request,
settings.HOME_NOT_LOGGED_IN)
# For users who have selected public access, we load the
# spectator experience. We fall through to the shared code
# for loading the application, with user_profile=None encoding
# that we're a spectator, not a logged-in user.
user_profile = None
update_last_reminder(user_profile)
statsd.incr("views.home")
# If a user hasn't signed the current Terms of Service, send them there
if need_accept_tos(user_profile):
return accounts_accept_terms(request)
narrow, narrow_stream, narrow_topic = detect_narrowed_window(
request, user_profile)
if user_profile is not None:
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = (first_in_realm
and not PreregistrationUser.objects.filter(
referred_by=user_profile).count())
needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING
else:
first_in_realm = False
prompt_for_invites = False
# The current tutorial doesn't super make sense for logged-out users.
needs_tutorial = False
queue_id, page_params = build_page_params_for_home_page_load(
request=request,
user_profile=user_profile,
realm=realm,
insecure_desktop_app=insecure_desktop_app,
narrow=narrow,
narrow_stream=narrow_stream,
narrow_topic=narrow_topic,
first_in_realm=first_in_realm,
prompt_for_invites=prompt_for_invites,
needs_tutorial=needs_tutorial,
)
log_data = RequestNotes.get_notes(request).log_data
assert log_data is not None
log_data["extra"] = f"[{queue_id}]"
csp_nonce = secrets.token_hex(24)
user_permission_info = get_user_permission_info(user_profile)
response = render(
request,
"zerver/app/index.html",
context={
"user_profile": user_profile,
"page_params": page_params,
"csp_nonce": csp_nonce,
"color_scheme": user_permission_info.color_scheme,
},
)
patch_cache_control(response,
no_cache=True,
no_store=True,
must_revalidate=True)
return response
def wrapped_func(*args: Any, **kwargs: Any) -> ReturnT:
ret = func(*args, **kwargs)
statsd.incr(counter, val)
return ret
def wrapped_func(*args, **kwargs):
ret = func(*args, **kwargs)
statsd.incr(counter, val)
return ret
def wrapped_func(*args: Any, **kwargs: Any) -> ReturnT:
ret = func(*args, **kwargs)
statsd.incr(counter, val)
return ret
def write_log_line(log_data,
path,
method,
remote_ip,
email,
client_name,
status_code=200,
error_content=None,
error_content_iter=None):
# type: (MutableMapping[str, Any], Text, str, str, Text, Text, int, Optional[AnyStr], Optional[Iterable[AnyStr]]) -> None
assert error_content is None or error_content_iter is None
if error_content is not None:
error_content_iter = (error_content, )
# For statsd timer name
if path == '/':
statsd_path = u'webreq'
else:
statsd_path = u"webreq.%s" % (path[1:].replace('/', '.'), )
# Remove non-ascii chars from path (there should be none, if there are it's
# because someone manually entered a nonexistant path), as UTF-8 chars make
# statsd sad when it sends the key name over the socket
statsd_path = statsd_path.encode('ascii',
errors='ignore').decode("ascii")
blacklisted_requests = [
'do_confirm', 'send_confirm', 'eventslast_event_id', 'webreq.content',
'avatar', 'user_uploads', 'password.reset', 'static', 'json.bots',
'json.users', 'json.streams', 'accounts.unsubscribe',
'apple-touch-icon', 'emoji', 'json.bots', 'upload_file',
'realm_activity', 'user_activity'
]
suppress_statsd = any(
(blacklisted in statsd_path for blacklisted in blacklisted_requests))
time_delta = -1
# A time duration of -1 means the StartLogRequests middleware
# didn't run for some reason
optional_orig_delta = ""
if 'time_started' in log_data:
time_delta = time.time() - log_data['time_started']
if 'time_stopped' in log_data:
orig_time_delta = time_delta
time_delta = ((log_data['time_stopped'] - log_data['time_started']) +
(time.time() - log_data['time_restarted']))
optional_orig_delta = " (lp: %s)" % (
format_timedelta(orig_time_delta), )
remote_cache_output = ""
if 'remote_cache_time_start' in log_data:
remote_cache_time_delta = get_remote_cache_time(
) - log_data['remote_cache_time_start']
remote_cache_count_delta = get_remote_cache_requests(
) - log_data['remote_cache_requests_start']
if 'remote_cache_requests_stopped' in log_data:
# (now - restarted) + (stopped - start) = (now - start) + (stopped - restarted)
remote_cache_time_delta += (
log_data['remote_cache_time_stopped'] -
log_data['remote_cache_time_restarted'])
remote_cache_count_delta += (
log_data['remote_cache_requests_stopped'] -
log_data['remote_cache_requests_restarted'])
if (remote_cache_time_delta > 0.005):
remote_cache_output = " (mem: %s/%s)" % (format_timedelta(
remote_cache_time_delta), remote_cache_count_delta)
if not suppress_statsd:
statsd.timing("%s.remote_cache.time" % (statsd_path, ),
timedelta_ms(remote_cache_time_delta))
statsd.incr("%s.remote_cache.querycount" % (statsd_path, ),
remote_cache_count_delta)
startup_output = ""
if 'startup_time_delta' in log_data and log_data[
"startup_time_delta"] > 0.005:
startup_output = " (+start: %s)" % (format_timedelta(
log_data["startup_time_delta"]))
bugdown_output = ""
if 'bugdown_time_start' in log_data:
bugdown_time_delta = get_bugdown_time(
) - log_data['bugdown_time_start']
bugdown_count_delta = get_bugdown_requests(
) - log_data['bugdown_requests_start']
if 'bugdown_requests_stopped' in log_data:
# (now - restarted) + (stopped - start) = (now - start) + (stopped - restarted)
bugdown_time_delta += (log_data['bugdown_time_stopped'] -
log_data['bugdown_time_restarted'])
bugdown_count_delta += (log_data['bugdown_requests_stopped'] -
log_data['bugdown_requests_restarted'])
if (bugdown_time_delta > 0.005):
bugdown_output = " (md: %s/%s)" % (
format_timedelta(bugdown_time_delta), bugdown_count_delta)
if not suppress_statsd:
statsd.timing("%s.markdown.time" % (statsd_path, ),
timedelta_ms(bugdown_time_delta))
statsd.incr("%s.markdown.count" % (statsd_path, ),
bugdown_count_delta)
# Get the amount of time spent doing database queries
db_time_output = ""
queries = connection.connection.queries if connection.connection is not None else []
if len(queries) > 0:
query_time = sum(float(query.get('time', 0)) for query in queries)
db_time_output = " (db: %s/%sq)" % (format_timedelta(query_time),
len(queries))
if not suppress_statsd:
# Log ms, db ms, and num queries to statsd
statsd.timing("%s.dbtime" % (statsd_path, ),
timedelta_ms(query_time))
statsd.incr("%s.dbq" % (statsd_path, ), len(queries))
statsd.timing("%s.total" % (statsd_path, ),
timedelta_ms(time_delta))
if 'extra' in log_data:
extra_request_data = " %s" % (log_data['extra'], )
else:
extra_request_data = ""
logger_client = "(%s via %s)" % (email, client_name)
logger_timing = ('%5s%s%s%s%s%s %s' %
(format_timedelta(time_delta), optional_orig_delta,
remote_cache_output, bugdown_output, db_time_output,
startup_output, path))
logger_line = ('%-15s %-7s %3d %s%s %s' %
(remote_ip, method, status_code, logger_timing,
extra_request_data, logger_client))
if (status_code in [200, 304] and method == "GET"
and path.startswith("/static")):
logger.debug(logger_line)
else:
logger.info(logger_line)
if (is_slow_query(time_delta, path)):
queue_json_publish("slow_queries", "%s (%s)" % (logger_line, email),
lambda e: None)
if settings.PROFILE_ALL_REQUESTS:
log_data["prof"].disable()
profile_path = "/tmp/profile.data.%s.%s" % (
path.split("/")[-1],
int(time_delta * 1000),
)
log_data["prof"].dump_stats(profile_path)
# Log some additional data whenever we return certain 40x errors
if 400 <= status_code < 500 and status_code not in [401, 404, 405]:
assert error_content_iter is not None
error_content_list = list(error_content_iter)
if error_content_list:
error_data = u''
elif isinstance(error_content_list[0], Text):
error_data = u''.join(error_content_list)
elif isinstance(error_content_list[0], binary_type):
error_data = repr(b''.join(error_content_list))
if len(error_data) > 100:
error_data = u"[content more than 100 characters]"
logger.info('status=%3d, data=%s, uid=%s' %
(status_code, error_data, email))
def home_real(request: HttpRequest) -> HttpResponse:
# Before we do any real work, check if the app is banned.
(insecure_desktop_app, banned_desktop_app,
auto_update_broken) = is_outdated_desktop_app(
request.META.get("HTTP_USER_AGENT", ""))
if banned_desktop_app:
return render(request,
'zerver/insecure_desktop_app.html',
context={
"auto_update_broken": auto_update_broken,
})
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
if request.user.is_authenticated:
user_profile = request.user
else: # nocoverage
# This code path should not be reachable because of zulip_login_required above.
user_profile = None
# If a user hasn't signed the current Terms of Service, send them there
if need_accept_tos(user_profile):
return accounts_accept_terms(request)
narrow, narrow_stream, narrow_topic = detect_narrowed_window(
request, user_profile)
register_ret = do_events_register(user_profile,
request.client,
apply_markdown=True,
client_gravatar=True,
slim_presence=True,
notification_settings_null=True,
narrow=narrow)
user_has_messages = (register_ret['max_message_id'] != -1)
update_last_reminder(user_profile)
if user_profile is not None:
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = (first_in_realm
and not PreregistrationUser.objects.filter(
referred_by=user_profile).count())
needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING
else: # nocoverage
first_in_realm = False
prompt_for_invites = False
# The current tutorial doesn't super make sense for logged-out users.
needs_tutorial = False
if user_profile is None: # nocoverage
furthest_read_time = time.time() # type: Optional[float]
elif user_profile.pointer == -1:
if user_has_messages:
# Put the new user's pointer at the bottom
#
# This improves performance, because we limit backfilling of messages
# before the pointer. It's also likely that someone joining an
# organization is interested in recent messages more than the very
# first messages on the system.
register_ret['pointer'] = register_ret['max_message_id']
user_profile.last_pointer_updater = request.session.session_key
furthest_read_time = None
else:
latest_read = get_usermessage_by_message_id(user_profile,
user_profile.pointer)
if latest_read is None:
# Don't completely fail if your saved pointer ID is invalid
logging.warning("User %s has invalid pointer %s" %
(user_profile.id, user_profile.pointer))
furthest_read_time = sent_time_in_epoch_seconds(latest_read)
# We pick a language for the user as follows:
# * First priority is the language in the URL, for debugging.
# * If not in the URL, we use the language from the user's settings.
request_language = translation.get_language_from_path(request.path_info)
if request_language is None:
request_language = register_ret['default_language']
translation.activate(request_language)
# We also save the language to the user's session, so that
# something reasonable will happen in logged-in portico pages.
request.session[
translation.LANGUAGE_SESSION_KEY] = translation.get_language()
two_fa_enabled = settings.TWO_FACTOR_AUTHENTICATION_ENABLED and user_profile is not None
# Pass parameters to the client-side JavaScript code.
# These end up in a global JavaScript Object named 'page_params'.
page_params = dict(
# Server settings.
development_environment=settings.DEVELOPMENT,
debug_mode=settings.DEBUG,
test_suite=settings.TEST_SUITE,
poll_timeout=settings.POLL_TIMEOUT,
insecure_desktop_app=insecure_desktop_app,
login_page=settings.HOME_NOT_LOGGED_IN,
root_domain_uri=settings.ROOT_DOMAIN_URI,
max_file_upload_size=settings.MAX_FILE_UPLOAD_SIZE,
max_avatar_file_size=settings.MAX_AVATAR_FILE_SIZE,
server_generation=settings.SERVER_GENERATION,
save_stacktraces=settings.SAVE_FRONTEND_STACKTRACES,
warn_no_email=settings.WARN_NO_EMAIL,
server_inline_image_preview=settings.INLINE_IMAGE_PREVIEW,
server_inline_url_embed_preview=settings.INLINE_URL_EMBED_PREVIEW,
password_min_length=settings.PASSWORD_MIN_LENGTH,
password_min_guesses=settings.PASSWORD_MIN_GUESSES,
jitsi_server_url=settings.JITSI_SERVER_URL,
search_pills_enabled=settings.SEARCH_PILLS_ENABLED,
server_avatar_changes_disabled=settings.AVATAR_CHANGES_DISABLED,
server_name_changes_disabled=settings.NAME_CHANGES_DISABLED,
# Misc. extra data.
have_initial_messages=user_has_messages,
initial_servertime=time.time(
), # Used for calculating relative presence age
default_language_name=get_language_name(
register_ret['default_language']),
language_list_dbl_col=get_language_list_for_templates(
register_ret['default_language']),
language_list=get_language_list(),
needs_tutorial=needs_tutorial,
first_in_realm=first_in_realm,
prompt_for_invites=prompt_for_invites,
furthest_read_time=furthest_read_time,
has_mobile_devices=user_profile is not None
and num_push_devices_for_user(user_profile) > 0,
bot_types=get_bot_types(user_profile),
two_fa_enabled=two_fa_enabled,
# Adding two_fa_enabled as condition saves us 3 queries when
# 2FA is not enabled.
two_fa_enabled_user=two_fa_enabled
and bool(default_device(user_profile)),
)
undesired_register_ret_fields = [
'streams',
]
for field_name in set(
register_ret.keys()) - set(undesired_register_ret_fields):
page_params[field_name] = register_ret[field_name]
if narrow_stream is not None:
# In narrow_stream context, initial pointer is just latest message
recipient = narrow_stream.recipient
try:
initial_pointer = Message.objects.filter(
recipient=recipient).order_by('id').reverse()[0].id
except IndexError:
initial_pointer = -1
page_params["narrow_stream"] = narrow_stream.name
if narrow_topic is not None:
page_params["narrow_topic"] = narrow_topic
page_params["narrow"] = [
dict(operator=term[0], operand=term[1]) for term in narrow
]
page_params["max_message_id"] = initial_pointer
page_params["pointer"] = initial_pointer
page_params["have_initial_messages"] = (initial_pointer != -1)
page_params["enable_desktop_notifications"] = False
statsd.incr('views.home')
show_invites, show_add_streams = compute_show_invites_and_add_streams(
user_profile)
show_billing = False
show_plans = False
if settings.CORPORATE_ENABLED and user_profile is not None:
from corporate.models import Customer, CustomerPlan
if user_profile.is_billing_admin or user_profile.is_realm_admin:
customer = Customer.objects.filter(
realm=user_profile.realm).first()
if customer is not None and CustomerPlan.objects.filter(
customer=customer).exists():
show_billing = True
if user_profile.realm.plan_type == Realm.LIMITED:
show_plans = True
request._log_data['extra'] = "[%s]" % (register_ret["queue_id"], )
page_params['translation_data'] = {}
if request_language != 'en':
page_params['translation_data'] = get_language_translation_data(
request_language)
csp_nonce = generate_random_token(48)
if user_profile is not None:
night_mode = user_profile.night_mode
is_guest = user_profile.is_guest
is_realm_admin = user_profile.is_realm_admin
show_webathena = user_profile.realm.webathena_enabled
else: # nocoverage
night_mode = False
is_guest = False
is_realm_admin = False
show_webathena = False
navbar_logo_url = compute_navbar_logo_url(page_params)
response = render(
request,
'zerver/app/index.html',
context={
'user_profile': user_profile,
'page_params': page_params,
'csp_nonce': csp_nonce,
'search_pills_enabled': settings.SEARCH_PILLS_ENABLED,
'show_invites': show_invites,
'show_add_streams': show_add_streams,
'show_billing': show_billing,
'show_plans': show_plans,
'is_admin': is_realm_admin,
'is_guest': is_guest,
'night_mode': night_mode,
'navbar_logo_url': navbar_logo_url,
'show_webathena': show_webathena,
'embedded': narrow_stream is not None,
'invite_as': PreregistrationUser.INVITE_AS,
'max_file_upload_size': settings.MAX_FILE_UPLOAD_SIZE,
},
)
patch_cache_control(response,
no_cache=True,
no_store=True,
must_revalidate=True)
return response
def get_old_messages_backend(request, user_profile,
anchor = REQ(converter=int),
num_before = REQ(converter=to_non_negative_int),
num_after = REQ(converter=to_non_negative_int),
narrow = REQ('narrow', converter=narrow_parameter, default=None),
use_first_unread_anchor = REQ(default=False, converter=ujson.loads),
apply_markdown=REQ(default=True,
converter=ujson.loads)):
# type: (HttpRequest, UserProfile, int, int, int, Optional[List[Dict[str, Any]]], bool, bool) -> HttpResponse
include_history = ok_to_include_history(narrow, user_profile.realm)
if include_history and not use_first_unread_anchor:
query = select([column("id").label("message_id")], None, table("zerver_message"))
inner_msg_id_col = literal_column("zerver_message.id")
elif narrow is None:
query = select([column("message_id"), column("flags")],
column("user_profile_id") == literal(user_profile.id),
table("zerver_usermessage"))
inner_msg_id_col = column("message_id")
else:
# TODO: Don't do this join if we're not doing a search
query = select([column("message_id"), column("flags")],
column("user_profile_id") == literal(user_profile.id),
join(table("zerver_usermessage"), table("zerver_message"),
literal_column("zerver_usermessage.message_id") ==
literal_column("zerver_message.id")))
inner_msg_id_col = column("message_id")
num_extra_messages = 1
is_search = False
if narrow is not None:
# Add some metadata to our logging data for narrows
verbose_operators = []
for term in narrow:
if term['operator'] == "is":
verbose_operators.append("is:" + term['operand'])
else:
verbose_operators.append(term['operator'])
request._log_data['extra'] = "[%s]" % (",".join(verbose_operators),)
# Build the query for the narrow
num_extra_messages = 0
builder = NarrowBuilder(user_profile, inner_msg_id_col)
search_term = None # type: Optional[Dict[str, Any]]
for term in narrow:
if term['operator'] == 'search':
if not is_search:
search_term = term
query = query.column(column("subject")).column(column("rendered_content"))
is_search = True
else:
# Join the search operators if there are multiple of them
search_term['operand'] += ' ' + term['operand']
else:
query = builder.add_term(query, term)
if is_search:
query = builder.add_term(query, search_term)
# We add 1 to the number of messages requested if no narrow was
# specified to ensure that the resulting list always contains the
# anchor message. If a narrow was specified, the anchor message
# might not match the narrow anyway.
if num_after != 0:
num_after += num_extra_messages
else:
num_before += num_extra_messages
sa_conn = get_sqlalchemy_connection()
if use_first_unread_anchor:
condition = column("flags").op("&")(UserMessage.flags.read.mask) == 0
# We exclude messages on muted topics when finding the first unread
# message in this narrow
muting_conditions = exclude_muting_conditions(user_profile, narrow)
if muting_conditions:
condition = and_(condition, *muting_conditions)
first_unread_query = query.where(condition)
first_unread_query = first_unread_query.order_by(inner_msg_id_col.asc()).limit(1)
first_unread_result = list(sa_conn.execute(first_unread_query).fetchall())
if len(first_unread_result) > 0:
anchor = first_unread_result[0][0]
else:
anchor = LARGER_THAN_MAX_MESSAGE_ID
before_query = None
after_query = None
if num_before != 0:
before_anchor = anchor
if num_after != 0:
# Don't include the anchor in both the before query and the after query
before_anchor = anchor - 1
before_query = query.where(inner_msg_id_col <= before_anchor) \
.order_by(inner_msg_id_col.desc()).limit(num_before)
if num_after != 0:
after_query = query.where(inner_msg_id_col >= anchor) \
.order_by(inner_msg_id_col.asc()).limit(num_after)
if anchor == LARGER_THAN_MAX_MESSAGE_ID:
# There's no need for an after_query if we're targeting just the target message.
after_query = None
if before_query is not None:
if after_query is not None:
query = union_all(before_query.self_group(), after_query.self_group())
else:
query = before_query
elif after_query is not None:
query = after_query
else:
# This can happen when a narrow is specified.
query = query.where(inner_msg_id_col == anchor)
main_query = alias(query)
query = select(main_query.c, None, main_query).order_by(column("message_id").asc())
# This is a hack to tag the query we use for testing
query = query.prefix_with("/* get_old_messages */")
query_result = list(sa_conn.execute(query).fetchall())
# The following is a little messy, but ensures that the code paths
# are similar regardless of the value of include_history. The
# 'user_messages' dictionary maps each message to the user's
# UserMessage object for that message, which we will attach to the
# rendered message dict before returning it. We attempt to
# bulk-fetch rendered message dicts from remote cache using the
# 'messages' list.
search_fields = dict() # type: Dict[int, Dict[str, Text]]
message_ids = [] # type: List[int]
user_message_flags = {} # type: Dict[int, List[str]]
if include_history:
message_ids = [row[0] for row in query_result]
# TODO: This could be done with an outer join instead of two queries
user_message_flags = dict((user_message.message_id, user_message.flags_list()) for user_message in
UserMessage.objects.filter(user_profile=user_profile,
message__id__in=message_ids))
for row in query_result:
message_id = row[0]
if user_message_flags.get(message_id) is None:
user_message_flags[message_id] = ["read", "historical"]
if is_search:
(_, subject, rendered_content, content_matches, subject_matches) = row
search_fields[message_id] = get_search_fields(rendered_content, subject,
content_matches, subject_matches)
else:
for row in query_result:
message_id = row[0]
flags = row[1]
user_message_flags[message_id] = parse_usermessage_flags(flags)
message_ids.append(message_id)
if is_search:
(_, _, subject, rendered_content, content_matches, subject_matches) = row
search_fields[message_id] = get_search_fields(rendered_content, subject,
content_matches, subject_matches)
cache_transformer = lambda row: MessageDict.build_dict_from_raw_db_row(row, apply_markdown)
id_fetcher = lambda row: row['id']
message_dicts = generic_bulk_cached_fetch(lambda message_id: to_dict_cache_key_id(message_id, apply_markdown),
Message.get_raw_db_rows,
message_ids,
id_fetcher=id_fetcher,
cache_transformer=cache_transformer,
extractor=extract_message_dict,
setter=stringify_message_dict)
message_list = []
for message_id in message_ids:
msg_dict = message_dicts[message_id]
msg_dict.update({"flags": user_message_flags[message_id]})
msg_dict.update(search_fields.get(message_id, {}))
message_list.append(msg_dict)
statsd.incr('loaded_old_messages', len(message_list))
ret = {'messages': message_list,
"result": "success",
"msg": ""}
return json_success(ret)
def wrapped_func(*args, **kwargs):
# type: (*Any, **Any) -> Any
ret = func(*args, **kwargs)
statsd.incr(counter, val)
return ret
def get_messages_backend(request: HttpRequest, user_profile: UserProfile,
anchor_val: Optional[str]=REQ(
'anchor', str_validator=check_string, default=None),
num_before: int=REQ(converter=to_non_negative_int),
num_after: int=REQ(converter=to_non_negative_int),
narrow: OptionalNarrowListT=REQ('narrow', converter=narrow_parameter, default=None),
use_first_unread_anchor_val: bool=REQ('use_first_unread_anchor',
validator=check_bool, default=False),
client_gravatar: bool=REQ(validator=check_bool, default=False),
apply_markdown: bool=REQ(validator=check_bool, default=True)) -> HttpResponse:
anchor = parse_anchor_value(anchor_val, use_first_unread_anchor_val)
if num_before + num_after > MAX_MESSAGES_PER_FETCH:
return json_error(_("Too many messages requested (maximum {}).").format(
MAX_MESSAGES_PER_FETCH,
))
if user_profile.realm.email_address_visibility != Realm.EMAIL_ADDRESS_VISIBILITY_EVERYONE:
# If email addresses are only available to administrators,
# clients cannot compute gravatars, so we force-set it to false.
client_gravatar = False
include_history = ok_to_include_history(narrow, user_profile)
if include_history:
# The initial query in this case doesn't use `zerver_usermessage`,
# and isn't yet limited to messages the user is entitled to see!
#
# This is OK only because we've made sure this is a narrow that
# will cause us to limit the query appropriately later.
# See `ok_to_include_history` for details.
need_message = True
need_user_message = False
elif narrow is None:
# We need to limit to messages the user has received, but we don't actually
# need any fields from Message
need_message = False
need_user_message = True
else:
need_message = True
need_user_message = True
query, inner_msg_id_col = get_base_query_for_search(
user_profile=user_profile,
need_message=need_message,
need_user_message=need_user_message,
)
query, is_search = add_narrow_conditions(
user_profile=user_profile,
inner_msg_id_col=inner_msg_id_col,
query=query,
narrow=narrow,
)
if narrow is not None:
# Add some metadata to our logging data for narrows
verbose_operators = []
for term in narrow:
if term['operator'] == "is":
verbose_operators.append("is:" + term['operand'])
else:
verbose_operators.append(term['operator'])
request._log_data['extra'] = "[{}]".format(",".join(verbose_operators))
sa_conn = get_sqlalchemy_connection()
if anchor is None:
# The use_first_unread_anchor code path
anchor = find_first_unread_anchor(
sa_conn,
user_profile,
narrow,
)
anchored_to_left = (anchor == 0)
# Set value that will be used to short circuit the after_query
# altogether and avoid needless conditions in the before_query.
anchored_to_right = (anchor >= LARGER_THAN_MAX_MESSAGE_ID)
if anchored_to_right:
num_after = 0
first_visible_message_id = get_first_visible_message_id(user_profile.realm)
query = limit_query_to_range(
query=query,
num_before=num_before,
num_after=num_after,
anchor=anchor,
anchored_to_left=anchored_to_left,
anchored_to_right=anchored_to_right,
id_col=inner_msg_id_col,
first_visible_message_id=first_visible_message_id,
)
main_query = alias(query)
query = select(main_query.c, None, main_query).order_by(column("message_id").asc())
# This is a hack to tag the query we use for testing
query = query.prefix_with("/* get_messages */")
rows = list(sa_conn.execute(query).fetchall())
query_info = post_process_limited_query(
rows=rows,
num_before=num_before,
num_after=num_after,
anchor=anchor,
anchored_to_left=anchored_to_left,
anchored_to_right=anchored_to_right,
first_visible_message_id=first_visible_message_id,
)
rows = query_info['rows']
# The following is a little messy, but ensures that the code paths
# are similar regardless of the value of include_history. The
# 'user_messages' dictionary maps each message to the user's
# UserMessage object for that message, which we will attach to the
# rendered message dict before returning it. We attempt to
# bulk-fetch rendered message dicts from remote cache using the
# 'messages' list.
message_ids: List[int] = []
user_message_flags: Dict[int, List[str]] = {}
if include_history:
message_ids = [row[0] for row in rows]
# TODO: This could be done with an outer join instead of two queries
um_rows = UserMessage.objects.filter(user_profile=user_profile,
message__id__in=message_ids)
user_message_flags = {um.message_id: um.flags_list() for um in um_rows}
for message_id in message_ids:
if message_id not in user_message_flags:
user_message_flags[message_id] = ["read", "historical"]
else:
for row in rows:
message_id = row[0]
flags = row[1]
user_message_flags[message_id] = UserMessage.flags_list_for_flags(flags)
message_ids.append(message_id)
search_fields: Dict[int, Dict[str, str]] = dict()
if is_search:
for row in rows:
message_id = row[0]
(topic_name, rendered_content, content_matches, topic_matches) = row[-4:]
try:
search_fields[message_id] = get_search_fields(rendered_content, topic_name,
content_matches, topic_matches)
except UnicodeDecodeError as err: # nocoverage
# No coverage for this block since it should be
# impossible, and we plan to remove it once we've
# debugged the case that makes it happen.
raise Exception(str(err), message_id, narrow)
message_list = messages_for_ids(
message_ids=message_ids,
user_message_flags=user_message_flags,
search_fields=search_fields,
apply_markdown=apply_markdown,
client_gravatar=client_gravatar,
allow_edit_history=user_profile.realm.allow_edit_history,
)
statsd.incr('loaded_old_messages', len(message_list))
ret = dict(
messages=message_list,
result='success',
msg='',
found_anchor=query_info['found_anchor'],
found_oldest=query_info['found_oldest'],
found_newest=query_info['found_newest'],
history_limited=query_info['history_limited'],
anchor=anchor,
)
return json_success(ret)
def home_real(request: HttpRequest) -> HttpResponse:
# Before we do any real work, check if the app is banned.
client_user_agent = request.META.get("HTTP_USER_AGENT", "")
(insecure_desktop_app, banned_desktop_app,
auto_update_broken) = is_outdated_desktop_app(client_user_agent)
if banned_desktop_app:
return render(
request,
"zerver/insecure_desktop_app.html",
context={
"auto_update_broken": auto_update_broken,
},
)
(unsupported_browser,
browser_name) = is_unsupported_browser(client_user_agent)
if unsupported_browser:
return render(
request,
"zerver/unsupported_browser.html",
context={
"browser_name": browser_name,
},
)
# We need to modify the session object every two weeks or it will expire.
# This line makes reloading the page a sufficient action to keep the
# session alive.
request.session.modified = True
if request.user.is_authenticated:
user_profile = request.user
realm = user_profile.realm
else:
# user_profile=None corresponds to the logged-out "web_public" visitor case.
user_profile = None
realm = get_valid_realm_from_request(request)
update_last_reminder(user_profile)
statsd.incr("views.home")
# If a user hasn't signed the current Terms of Service, send them there
if need_accept_tos(user_profile):
return accounts_accept_terms(request)
narrow, narrow_stream, narrow_topic = detect_narrowed_window(
request, user_profile)
if user_profile is not None:
first_in_realm = realm_user_count(user_profile.realm) == 1
# If you are the only person in the realm and you didn't invite
# anyone, we'll continue to encourage you to do so on the frontend.
prompt_for_invites = (first_in_realm
and not PreregistrationUser.objects.filter(
referred_by=user_profile).count())
needs_tutorial = user_profile.tutorial_status == UserProfile.TUTORIAL_WAITING
else:
first_in_realm = False
prompt_for_invites = False
# The current tutorial doesn't super make sense for logged-out users.
needs_tutorial = False
queue_id, page_params = build_page_params_for_home_page_load(
request=request,
user_profile=user_profile,
realm=realm,
insecure_desktop_app=insecure_desktop_app,
narrow=narrow,
narrow_stream=narrow_stream,
narrow_topic=narrow_topic,
first_in_realm=first_in_realm,
prompt_for_invites=prompt_for_invites,
needs_tutorial=needs_tutorial,
)
show_invites, show_add_streams = compute_show_invites_and_add_streams(
user_profile)
request._log_data["extra"] = "[{}]".format(queue_id)
csp_nonce = secrets.token_hex(24)
user_permission_info = get_user_permission_info(user_profile)
response = render(
request,
"zerver/app/index.html",
context={
"user_profile": user_profile,
"page_params": page_params,
"csp_nonce": csp_nonce,
"search_pills_enabled": settings.SEARCH_PILLS_ENABLED,
"show_invites": show_invites,
"show_add_streams": show_add_streams,
"is_owner": user_permission_info.is_realm_owner,
"is_admin": user_permission_info.is_realm_admin,
"is_guest": user_permission_info.is_guest,
"color_scheme": user_permission_info.color_scheme,
"embedded": narrow_stream is not None,
"max_file_upload_size_mib": settings.MAX_FILE_UPLOAD_SIZE,
},
)
patch_cache_control(response,
no_cache=True,
no_store=True,
must_revalidate=True)
return response
