def report_send_times(
request: HttpRequest,
user_profile: UserProfile,
time: int = REQ(converter=to_non_negative_int),
received: int = REQ(converter=to_non_negative_int, default=-1),
displayed: int = REQ(converter=to_non_negative_int, default=-1),
locally_echoed: bool = REQ(json_validator=check_bool, default=False),
rendered_content_disparity: bool = REQ(json_validator=check_bool, default=False),
) -> HttpResponse:
received_str = "(unknown)"
if received > 0:
received_str = str(received)
displayed_str = "(unknown)"
if displayed > 0:
displayed_str = str(displayed)
log_data = get_request_notes(request).log_data
assert log_data is not None
log_data[
"extra"
] = f"[{time}ms/{received_str}ms/{displayed_str}ms/echo:{locally_echoed}/diff:{rendered_content_disparity}]"
base_key = statsd_key(user_profile.realm.string_id, clean_periods=True)
statsd.timing(f"endtoend.send_time.{base_key}", time)
if received > 0:
statsd.timing(f"endtoend.receive_time.{base_key}", received)
if displayed > 0:
statsd.timing(f"endtoend.displayed_time.{base_key}", displayed)
if locally_echoed:
statsd.incr("locally_echoed")
if rendered_content_disparity:
statsd.incr("render_disparity")
return json_success()
def api_slack_webhook(
request: HttpRequest,
user_profile: UserProfile,
user_name: str = REQ(),
text: str = REQ(),
channel_name: str = REQ(),
stream: str = REQ(default="slack"),
channels_map_to_topics: str = REQ(default="1"),
) -> HttpResponse:
if channels_map_to_topics not in list(VALID_OPTIONS.values()):
raise JsonableError(
_("Error: channels_map_to_topics parameter other than 0 or 1"))
if channels_map_to_topics == VALID_OPTIONS["SHOULD_BE_MAPPED"]:
subject = f"channel: {channel_name}"
else:
stream = channel_name
subject = _("Message from Slack")
content = ZULIP_MESSAGE_TEMPLATE.format(message_sender=user_name,
text=text)
client = get_request_notes(request).client
assert client is not None
check_send_stream_message(user_profile, client, stream, subject, content)
return json_success()
def api_dialogflow_webhook(
request: HttpRequest,
user_profile: UserProfile,
payload: Dict[str, Any] = REQ(argument_type="body"),
email: str = REQ(),
) -> HttpResponse:
status = payload["status"]["code"]
if status == 200:
result = payload["result"]["fulfillment"]["speech"]
if not result:
alternate_result = payload["alternateResult"]["fulfillment"][
"speech"]
if not alternate_result:
body = "Dialogflow couldn't process your query."
else:
body = alternate_result
else:
body = result
else:
error_status = payload["status"]["errorDetails"]
body = f"{status} - {error_status}"
receiving_user = get_user(email, user_profile.realm)
client = get_request_notes(request).client
assert client is not None
check_send_private_message(user_profile, client, receiving_user, body)
return json_success()
def process_client(
request: HttpRequest,
user_profile: UserProfile,
*,
is_browser_view: bool = False,
client_name: Optional[str] = None,
skip_update_user_activity: bool = False,
query: Optional[str] = None,
) -> None:
request_notes = get_request_notes(request)
if client_name is None:
client_name = request_notes.client_name
assert client_name is not None
# We could check for a browser's name being "Mozilla", but
# e.g. Opera and MobileSafari don't set that, and it seems
# more robust to just key off whether it was a browser view
if is_browser_view and not client_name.startswith("Zulip"):
# Avoid changing the client string for browsers, but let
# the Zulip desktop apps be themselves.
client_name = "website"
request_notes.client = get_client(client_name)
if not skip_update_user_activity and user_profile.is_authenticated:
update_user_activity(request, user_profile, query)
def process_view(
self,
request: HttpRequest,
view_func: ViewFuncT,
args: List[str],
kwargs: Dict[str, Any],
) -> None:
request_notes = get_request_notes(request)
if request_notes.saved_response is not None:
# The below logging adjustments are unnecessary (because
# we've already imported everything) and incorrect
# (because they'll overwrite data from pre-long-poll
# request processing) when returning a saved response.
return
# process_request was already run; we save the initialization
# time (i.e. the time between receiving the request and
# figuring out which view function to call, which is primarily
# importing modules on the first start)
assert request_notes.log_data is not None
request_notes.log_data["startup_time_delta"] = (
time.time() - request_notes.log_data["time_started"])
# And then completely reset our tracking to only cover work
# done as part of this request
record_request_start_data(request_notes.log_data)
def process_response(self, request: HttpRequest,
response: HttpResponseBase) -> HttpResponseBase:
# This is the same as the default LocaleMiddleware, minus the
# logic that redirects 404's that lack a prefixed language in
# the path into having a language. See
# https://code.djangoproject.com/ticket/32005
language = translation.get_language()
language_from_path = translation.get_language_from_path(
request.path_info)
urlconf = getattr(request, "urlconf", settings.ROOT_URLCONF)
i18n_patterns_used, _ = is_language_prefix_patterns_used(urlconf)
if not (i18n_patterns_used and language_from_path):
patch_vary_headers(response, ("Accept-Language", ))
assert language is not None
response.setdefault("Content-Language", language)
# An additional responsibility of our override of this middleware is to save the user's language
# preference in a cookie. That determination is made by code handling the request
# and saved in the set_language flag so that it can be used here.
set_language = get_request_notes(request).set_language
if set_language is not None:
response.set_cookie(settings.LANGUAGE_COOKIE_NAME, set_language)
return response
def process_exception(self, request: HttpRequest,
exception: Exception) -> Optional[HttpResponse]:
if isinstance(exception, MissingAuthenticationError):
if "text/html" in request.META.get("HTTP_ACCEPT", ""):
# If this looks like a request from a top-level page in a
# browser, send the user to the login page.
#
# TODO: The next part is a bit questionable; it will
# execute the likely intent for intentionally visiting
# an API endpoint without authentication in a browser,
# but that's an unlikely to be done intentionally often.
return HttpResponseRedirect(
f"{settings.HOME_NOT_LOGGED_IN}?next={request.path}")
if request.path.startswith("/api"):
# For API routes, ask for HTTP basic auth (email:apiKey).
return json_unauthorized()
else:
# For /json routes, ask for session authentication.
return json_unauthorized(www_authenticate="session")
if isinstance(exception, JsonableError):
return json_response_from_error(exception)
if get_request_notes(request).error_format == "JSON":
capture_exception(exception)
json_error_logger = logging.getLogger(
"zerver.middleware.json_error_handler")
json_error_logger.error(traceback.format_exc(),
extra=dict(request=request))
return json_response(res_type="error",
msg=_("Internal server error"),
status=500)
return None
def finish_mobile_flow(request: HttpRequest, user_profile: UserProfile,
otp: str) -> HttpResponse:
# For the mobile OAuth flow, we send the API key and other
# necessary details in a redirect to a zulip:// URI scheme.
api_key = get_api_key(user_profile)
response = create_response_for_otp_flow(
api_key,
otp,
user_profile,
encrypted_key_field_name="otp_encrypted_api_key")
# Since we are returning an API key instead of going through
# the Django login() function (which creates a browser
# session, etc.), the "new login" signal handler (which
# triggers an email notification new logins) will not run
# automatically. So we call it manually here.
#
# Arguably, sending a fake 'user_logged_in' signal would be a better approach:
# user_logged_in.send(sender=user_profile.__class__, request=request, user=user_profile)
email_on_new_login(sender=user_profile.__class__,
request=request,
user=user_profile)
# Mark this request as having a logged-in user for our server logs.
process_client(request, user_profile)
get_request_notes(
request).requestor_for_logs = user_profile.format_requestor_for_logs()
return response
def get_events_internal(
request: HttpRequest, user_profile_id: int = REQ(json_validator=check_int)
) -> HttpResponse:
user_profile = get_user_profile_by_id(user_profile_id)
get_request_notes(request).requestor_for_logs = user_profile.format_requestor_for_logs()
process_client(request, user_profile, client_name="internal")
return get_events_backend(request, user_profile)
def mark_topic_as_read(
request: HttpRequest,
user_profile: UserProfile,
stream_id: int = REQ(json_validator=check_int),
topic_name: str = REQ(),
) -> HttpResponse:
stream, sub = access_stream_by_id(user_profile, stream_id)
if topic_name:
topic_exists = user_message_exists_for_topic(
user_profile=user_profile,
recipient_id=stream.recipient_id,
topic_name=topic_name,
)
if not topic_exists:
raise JsonableError(_("No such topic '{}'").format(topic_name))
count = do_mark_stream_messages_as_read(user_profile, stream.recipient_id,
topic_name)
log_data_str = f"[{count} updated]"
log_data = get_request_notes(request).log_data
assert log_data is not None
log_data["extra"] = log_data_str
return json_success({"result": "success", "msg": ""})
def update_message_backend(
request: HttpRequest,
user_profile: UserProfile,
message_id: int = REQ(converter=to_non_negative_int, path_only=True),
stream_id: Optional[int] = REQ(converter=to_non_negative_int, default=None),
topic_name: Optional[str] = REQ_topic(),
propagate_mode: str = REQ(
default="change_one", str_validator=check_string_in(PROPAGATE_MODE_VALUES)
),
send_notification_to_old_thread: bool = REQ(default=True, json_validator=check_bool),
send_notification_to_new_thread: bool = REQ(default=True, json_validator=check_bool),
content: Optional[str] = REQ(default=None),
) -> HttpResponse:
number_changed = check_update_message(
user_profile,
message_id,
stream_id,
topic_name,
propagate_mode,
send_notification_to_old_thread,
send_notification_to_new_thread,
content,
)
# Include the number of messages changed in the logs
log_data = get_request_notes(request).log_data
assert log_data is not None
log_data["extra"] = f"[{number_changed}]"
return json_success()
def async_request_timer_restart(request: HttpRequest) -> None:
log_data = get_request_notes(request).log_data
assert log_data is not None
if "time_restarted" in log_data:
# Don't destroy data when being called from
# finish_current_handler
return
record_request_restart_data(log_data)
def get_realm_from_request(request: HttpRequest) -> Optional[Realm]:
request_notes = get_request_notes(request)
if hasattr(request, "user") and hasattr(request.user, "realm"):
return request.user.realm
if not request_notes.has_fetched_realm:
# We cache the realm object from this function on the request data,
# so that functions that call get_realm_from_request don't
# need to do duplicate queries on the same realm while
# processing a single request.
subdomain = get_subdomain(request)
request_notes = get_request_notes(request)
try:
request_notes.realm = get_realm(subdomain)
except Realm.DoesNotExist:
request_notes.realm = None
request_notes.has_fetched_realm = True
return request_notes.realm
def client_is_exempt_from_rate_limiting(request: HttpRequest) -> bool:
# Don't rate limit requests from Django that come from our own servers,
# and don't rate-limit dev instances
client = get_request_notes(request).client
return (client is not None and client.name.lower() == "internal") and (
is_local_addr(request.META["REMOTE_ADDR"]) or settings.DEBUG_RATE_LIMITING
)
def alter_content(request: HttpRequest, content: bytes) -> bytes:
first_paragraph_text = get_content_description(content, request)
placeholder_open_graph_description = get_request_notes(
request).placeholder_open_graph_description
assert placeholder_open_graph_description is not None
return content.replace(
placeholder_open_graph_description.encode(),
first_paragraph_text.encode(),
)
def process_response(
self, request: HttpRequest,
response: StreamingHttpResponse) -> StreamingHttpResponse:
if get_request_notes(
request).placeholder_open_graph_description is not None:
assert not response.streaming
response.content = alter_content(request, response.content)
return response
def do_login(request: HttpRequest, user_profile: UserProfile) -> None:
"""Creates a session, logging in the user, using the Django method,
and also adds helpful data needed by our server logs.
"""
django_login(request, user_profile)
get_request_notes(request).requestor_for_logs = user_profile.format_requestor_for_logs()
process_client(request, user_profile, is_browser_view=True)
if settings.TWO_FACTOR_AUTHENTICATION_ENABLED:
# Log in with two factor authentication as well.
do_two_factor_login(request, user_profile)
def process_response(self, request: HttpRequest,
response: HttpResponse) -> HttpResponse:
if not settings.RATE_LIMITING:
return response
# Add X-RateLimit-*** headers
ratelimits_applied = get_request_notes(request).ratelimits_applied
if len(ratelimits_applied) > 0:
self.set_response_headers(response, ratelimits_applied)
return response
def mark_all_as_read(request: HttpRequest,
user_profile: UserProfile) -> HttpResponse:
request_notes = get_request_notes(request)
assert request_notes.client is not None
count = do_mark_all_as_read(user_profile, request_notes.client)
log_data_str = f"[{count} updated]"
assert request_notes.log_data is not None
request_notes.log_data["extra"] = log_data_str
return json_success({"result": "success", "msg": ""})
def render_message_backend(
request: HttpRequest, user_profile: UserProfile, content: str = REQ()
) -> HttpResponse:
message = Message()
message.sender = user_profile
message.content = content
client = get_request_notes(request).client
assert client is not None
message.sending_client = client
rendering_result = render_markdown(message, content, realm=user_profile.realm)
return json_success({"rendered": rendering_result.rendered_content})
def get_web_public_topics_backend(request: HttpRequest,
stream_id: int) -> HttpResponse:
try:
realm = get_request_notes(request).realm
assert realm is not None
stream = access_web_public_stream(stream_id, realm)
except JsonableError:
return json_success(dict(topics=[]))
result = get_topic_history_for_public_stream(
recipient_id=stream.recipient_id)
return json_success(dict(topics=result))
def cleanup_event_queue(
request: HttpRequest, user_profile: UserProfile, queue_id: str = REQ()
) -> HttpResponse:
client = get_client_descriptor(str(queue_id))
if client is None:
raise BadEventQueueIdError(queue_id)
if user_profile.id != client.user_profile_id:
raise JsonableError(_("You are not authorized to access this queue"))
log_data = get_request_notes(request).log_data
assert log_data is not None
log_data["extra"] = f"[{queue_id}]"
client.cleanup()
return json_success()
def report_unnarrow_times(
request: HttpRequest,
user_profile: Union[UserProfile, AnonymousUser],
initial_core: int = REQ(converter=to_non_negative_int),
initial_free: int = REQ(converter=to_non_negative_int),
) -> HttpResponse:
log_data = get_request_notes(request).log_data
assert log_data is not None
log_data["extra"] = f"[{initial_core}ms/{initial_free}ms]"
realm = get_valid_realm_from_request(request)
base_key = statsd_key(realm.string_id, clean_periods=True)
statsd.timing(f"unnarrow.initial_core.{base_key}", initial_core)
statsd.timing(f"unnarrow.initial_free.{base_key}", initial_free)
return json_success()
def mark_stream_as_read(
request: HttpRequest,
user_profile: UserProfile,
stream_id: int = REQ(json_validator=check_int)
) -> HttpResponse:
stream, sub = access_stream_by_id(user_profile, stream_id)
count = do_mark_stream_messages_as_read(user_profile, stream.recipient_id)
log_data_str = f"[{count} updated]"
log_data = get_request_notes(request).log_data
assert log_data is not None
log_data["extra"] = log_data_str
return json_success({"result": "success", "msg": ""})
def api_fetch_api_key(
request: HttpRequest, username: str = REQ(), password: str = REQ()
) -> HttpResponse:
return_data: Dict[str, bool] = {}
realm = get_realm_from_request(request)
if realm is None:
raise InvalidSubdomainError()
if not ldap_auth_enabled(realm=realm):
# In case we don't authenticate against LDAP, check for a valid
# email. LDAP backend can authenticate against a non-email.
validate_login_email(username)
user_profile = authenticate(request=request,
username=username,
password=password,
realm=realm,
return_data=return_data)
if return_data.get("inactive_user"):
raise UserDeactivatedError()
if return_data.get("inactive_realm"):
raise RealmDeactivatedError()
if return_data.get("password_auth_disabled"):
raise PasswordAuthDisabledError()
if return_data.get("password_reset_needed"):
raise PasswordResetRequiredError()
if user_profile is None:
raise AuthenticationFailedError()
assert user_profile.is_authenticated
# Maybe sending 'user_logged_in' signal is the better approach:
# user_logged_in.send(sender=user_profile.__class__, request=request, user=user_profile)
# Not doing this only because over here we don't add the user information
# in the session. If the signal receiver assumes that we do then that
# would cause problems.
email_on_new_login(sender=user_profile.__class__,
request=request,
user=user_profile)
# Mark this request as having a logged-in user for our server logs.
process_client(request, user_profile)
get_request_notes(
request).requestor_for_logs = user_profile.format_requestor_for_logs()
api_key = get_api_key(user_profile)
return json_success({
"api_key": api_key,
"email": user_profile.delivery_email
})
def api_yo_app_webhook(
request: HttpRequest,
user_profile: UserProfile,
email: str = REQ(default=""),
username: str = REQ(default="Yo Bot"),
topic: Optional[str] = REQ(default=None),
user_ip: Optional[str] = REQ(default=None),
) -> HttpResponse:
body = f"Yo from {username}"
receiving_user = get_user(email, user_profile.realm)
client = get_request_notes(request).client
assert client is not None
check_send_private_message(user_profile, client, receiving_user, body)
return json_success()
def _wrapped_func_arguments(
request: HttpRequest, *args: object, **kwargs: object
) -> HttpResponse:
if not authenticate_notify(request):
raise AccessDeniedError()
is_tornado_request = hasattr(request, "_tornado_handler")
# These next 2 are not security checks; they are internal
# assertions to help us find bugs.
if is_tornado_view and not is_tornado_request:
raise RuntimeError("Tornado notify view called with no Tornado handler")
if not is_tornado_view and is_tornado_request:
raise RuntimeError("Django notify view called with Tornado handler")
request_notes = get_request_notes(request)
request_notes.requestor_for_logs = "internal"
return view_func(request, *args, **kwargs)
def wrapper(request: HttpRequest, *args: object, **kwargs: object) -> HttpResponse:
if request.method != "POST":
err_method = request.method
logging.warning(
"Method Not Allowed (%s): %s",
err_method,
request.path,
extra={"status_code": 405, "request": request},
)
if get_request_notes(request).error_format == "JSON":
return json_method_not_allowed(["POST"])
else:
return TemplateResponse(
request, "404.html", context={"status_code": 405}, status=405
)
return func(request, *args, **kwargs)
def process_request(self, request: HttpRequest) -> None:
maybe_tracemalloc_listen()
request_notes = get_request_notes(request)
if request_notes.log_data is not None:
# Sanity check to ensure this is being called from the
# Tornado code path that returns responses asynchronously.
assert request_notes.saved_response is not None
# Avoid re-initializing request_notes.log_data if it's already there.
return
request_notes.client_name, request_notes.client_version = parse_client(
request)
request_notes.log_data = {}
record_request_start_data(request_notes.log_data)
def format(self, record: logging.LogRecord) -> str:
from zerver.lib.request import get_current_request
request = get_current_request()
if not request:
setattr(record, "user", None)
setattr(record, "client", None)
setattr(record, "url", None)
setattr(record, "content_type", None)
setattr(record, "custom_headers", None)
setattr(record, "payload", None)
return super().format(record)
if request.content_type == "application/json":
payload = request.body
else:
payload = request.POST.get("payload")
try:
payload = orjson.dumps(orjson.loads(payload),
option=orjson.OPT_INDENT_2).decode()
except orjson.JSONDecodeError:
pass
custom_header_template = "{header}: {value}\n"
header_text = ""
for header in request.META.keys():
if header.lower().startswith("http_x"):
header_text += custom_header_template.format(
header=header, value=request.META[header])
header_message = header_text if header_text else None
from zerver.lib.request import get_request_notes
client = get_request_notes(request).client
assert client is not None
setattr(
record, "user",
f"{request.user.delivery_email} ({request.user.realm.string_id})")
setattr(record, "client", client.name)
setattr(record, "url", request.META.get("PATH_INFO", None))
setattr(record, "content_type", request.content_type)
setattr(record, "custom_headers", header_message)
setattr(record, "payload", payload)
return super().format(record)