def process_exception(
self, request: HttpRequest, exception: Exception
) -> Optional[HttpResponse]:
if isinstance(exception, MissingAuthenticationError):
if "text/html" in request.META.get("HTTP_ACCEPT", ""):
# If this looks like a request from a top-level page in a
# browser, send the user to the login page.
#
# TODO: The next part is a bit questionable; it will
# execute the likely intent for intentionally visiting
# an API endpoint without authentication in a browser,
# but that's an unlikely to be done intentionally often.
return HttpResponseRedirect(f"{settings.HOME_NOT_LOGGED_IN}?next={request.path}")
if request.path.startswith("/api"):
# For API routes, ask for HTTP basic auth (email:apiKey).
return json_unauthorized()
else:
# For /json routes, ask for session authentication.
return json_unauthorized(www_authenticate="session")
if isinstance(exception, JsonableError):
return json_response_from_error(exception)
if RequestNotes.get_notes(request).error_format == "JSON":
capture_exception(exception)
json_error_logger = logging.getLogger("zerver.middleware.json_error_handler")
json_error_logger.error(traceback.format_exc(), extra=dict(request=request))
return json_response(res_type="error", msg=_("Internal server error"), status=500)
return None
async def convert_tornado_request_to_django_request(self) -> HttpRequest:
# This takes the WSGI environment that Tornado received (which
# fully describes the HTTP request that was sent to Tornado)
# and pass it to Django's WSGIRequest to generate a Django
# HttpRequest object with the original Tornado request's HTTP
# headers, parameters, etc.
environ = WSGIContainer.environ(self.request)
environ["PATH_INFO"] = urllib.parse.unquote(environ["PATH_INFO"])
# Django WSGIRequest setup code that should match logic from
# Django's WSGIHandler.__call__ before the call to
# `get_response()`.
set_script_prefix(get_script_name(environ))
await sync_to_async(
lambda: signals.request_started.send(sender=self.__class__),
thread_sensitive=True)()
self._request = WSGIRequest(environ)
# We do the import during runtime to avoid cyclic dependency
from zerver.lib.request import RequestNotes
# Provide a way for application code to access this handler
# given the HttpRequest object.
RequestNotes.get_notes(
self._request).tornado_handler_id = self.handler_id
return self._request
def process_client(
request: HttpRequest,
user: Union[UserProfile, AnonymousUser],
*,
is_browser_view: bool = False,
client_name: Optional[str] = None,
skip_update_user_activity: bool = False,
query: Optional[str] = None,
) -> None:
request_notes = RequestNotes.get_notes(request)
if client_name is None:
client_name = request_notes.client_name
assert client_name is not None
# We could check for a browser's name being "Mozilla", but
# e.g. Opera and MobileSafari don't set that, and it seems
# more robust to just key off whether it was a browser view
if is_browser_view and not client_name.startswith("Zulip"):
# Avoid changing the client string for browsers, but let
# the Zulip desktop apps be themselves.
client_name = "website"
request_notes.client = get_client(client_name)
if not skip_update_user_activity and user.is_authenticated:
update_user_activity(request, user, query)
def api_slack_webhook(
request: HttpRequest,
user_profile: UserProfile,
user_name: str = REQ(),
text: str = REQ(),
channel_name: str = REQ(),
stream: str = REQ(default="slack"),
channels_map_to_topics: str = REQ(default="1"),
) -> HttpResponse:
if channels_map_to_topics not in list(VALID_OPTIONS.values()):
raise JsonableError(
_("Error: channels_map_to_topics parameter other than 0 or 1"))
if channels_map_to_topics == VALID_OPTIONS["SHOULD_BE_MAPPED"]:
subject = f"channel: {channel_name}"
else:
stream = channel_name
subject = _("Message from Slack")
content = ZULIP_MESSAGE_TEMPLATE.format(message_sender=user_name,
text=text)
client = RequestNotes.get_notes(request).client
assert client is not None
check_send_stream_message(user_profile, client, stream, subject, content)
return json_success(request)
def process_view(
self,
request: HttpRequest,
view_func: ViewFuncT,
args: List[str],
kwargs: Dict[str, Any],
) -> None:
request_notes = RequestNotes.get_notes(request)
if request_notes.saved_response is not None:
# The below logging adjustments are unnecessary (because
# we've already imported everything) and incorrect
# (because they'll overwrite data from pre-long-poll
# request processing) when returning a saved response.
return
# process_request was already run; we save the initialization
# time (i.e. the time between receiving the request and
# figuring out which view function to call, which is primarily
# importing modules on the first start)
assert request_notes.log_data is not None
request_notes.log_data["startup_time_delta"] = (
time.time() - request_notes.log_data["time_started"]
)
# And then completely reset our tracking to only cover work
# done as part of this request
record_request_start_data(request_notes.log_data)
def update_message_backend(
request: HttpRequest,
user_profile: UserProfile,
message_id: int = REQ(converter=to_non_negative_int, path_only=True),
stream_id: Optional[int] = REQ(converter=to_non_negative_int,
default=None),
topic_name: Optional[str] = REQ_topic(),
propagate_mode: str = REQ(
default="change_one",
str_validator=check_string_in(PROPAGATE_MODE_VALUES)),
send_notification_to_old_thread: bool = REQ(default=True,
json_validator=check_bool),
send_notification_to_new_thread: bool = REQ(default=True,
json_validator=check_bool),
content: Optional[str] = REQ(default=None),
) -> HttpResponse:
number_changed = check_update_message(
user_profile,
message_id,
stream_id,
topic_name,
propagate_mode,
send_notification_to_old_thread,
send_notification_to_new_thread,
content,
)
# Include the number of messages changed in the logs
log_data = RequestNotes.get_notes(request).log_data
assert log_data is not None
log_data["extra"] = f"[{number_changed}]"
return json_success(request)
def finish_mobile_flow(request: HttpRequest, user_profile: UserProfile,
otp: str) -> HttpResponse:
# For the mobile OAuth flow, we send the API key and other
# necessary details in a redirect to a zulip:// URI scheme.
api_key = get_api_key(user_profile)
response = create_response_for_otp_flow(
api_key,
otp,
user_profile,
encrypted_key_field_name="otp_encrypted_api_key")
# Since we are returning an API key instead of going through
# the Django login() function (which creates a browser
# session, etc.), the "new login" signal handler (which
# triggers an email notification new logins) will not run
# automatically. So we call it manually here.
#
# Arguably, sending a fake 'user_logged_in' signal would be a better approach:
# user_logged_in.send(sender=user_profile.__class__, request=request, user=user_profile)
email_on_new_login(sender=user_profile.__class__,
request=request,
user=user_profile)
# Mark this request as having a logged-in user for our server logs.
process_client(request, user_profile)
RequestNotes.get_notes(
request).requestor_for_logs = user_profile.format_requestor_for_logs()
return response
def api_dialogflow_webhook(
request: HttpRequest,
user_profile: UserProfile,
payload: WildValue = REQ(argument_type="body",
converter=to_wild_value),
email: str = REQ(),
) -> HttpResponse:
status = payload["status"]["code"].tame(check_int)
if status == 200:
result = payload["result"]["fulfillment"]["speech"].tame(check_string)
if not result:
alternate_result = payload["alternateResult"]["fulfillment"][
"speech"].tame(check_string)
if not alternate_result:
body = "Dialogflow couldn't process your query."
else:
body = alternate_result
else:
body = result
else:
error_status = payload["status"]["errorDetails"].tame(check_string)
body = f"{status} - {error_status}"
receiving_user = get_user(email, user_profile.realm)
client = RequestNotes.get_notes(request).client
assert client is not None
check_send_private_message(user_profile, client, receiving_user, body)
return json_success(request)
def mark_topic_as_read(
request: HttpRequest,
user_profile: UserProfile,
stream_id: int = REQ(json_validator=check_int),
topic_name: str = REQ(),
) -> HttpResponse:
stream, sub = access_stream_by_id(user_profile, stream_id)
if topic_name:
topic_exists = user_message_exists_for_topic(
user_profile=user_profile,
recipient_id=stream.recipient_id,
topic_name=topic_name,
)
if not topic_exists:
raise JsonableError(_("No such topic '{}'").format(topic_name))
count = do_mark_stream_messages_as_read(user_profile, stream.recipient_id,
topic_name)
log_data_str = f"[{count} updated]"
log_data = RequestNotes.get_notes(request).log_data
assert log_data is not None
log_data["extra"] = log_data_str
return json_success(request)
def get_events_internal(
request: HttpRequest, user_profile_id: int = REQ(json_validator=check_int)
) -> HttpResponse:
user_profile = get_user_profile_by_id(user_profile_id)
RequestNotes.get_notes(request).requestor_for_logs = user_profile.format_requestor_for_logs()
process_client(request, user_profile, client_name="internal")
return get_events_backend(request, user_profile)
def report_send_times(
request: HttpRequest,
user_profile: UserProfile,
time: int = REQ(converter=to_non_negative_int),
received: int = REQ(converter=to_non_negative_int, default=-1),
displayed: int = REQ(converter=to_non_negative_int, default=-1),
locally_echoed: bool = REQ(json_validator=check_bool, default=False),
rendered_content_disparity: bool = REQ(json_validator=check_bool,
default=False),
) -> HttpResponse:
received_str = "(unknown)"
if received > 0:
received_str = str(received)
displayed_str = "(unknown)"
if displayed > 0:
displayed_str = str(displayed)
log_data = RequestNotes.get_notes(request).log_data
assert log_data is not None
log_data[
"extra"] = f"[{time}ms/{received_str}ms/{displayed_str}ms/echo:{locally_echoed}/diff:{rendered_content_disparity}]"
base_key = statsd_key(user_profile.realm.string_id, clean_periods=True)
statsd.timing(f"endtoend.send_time.{base_key}", time)
if received > 0:
statsd.timing(f"endtoend.receive_time.{base_key}", received)
if displayed > 0:
statsd.timing(f"endtoend.displayed_time.{base_key}", displayed)
if locally_echoed:
statsd.incr("locally_echoed")
if rendered_content_disparity:
statsd.incr("render_disparity")
return json_success(request)
def validate_scim_bearer_token(request: HttpRequest) -> Optional[SCIMClient]:
"""
This function verifies the request is allowed to make SCIM requests on this subdomain,
by checking the provided bearer token and ensuring it matches a scim client configured
for this subdomain in settings.SCIM_CONFIG.
If successful, returns the corresponding SCIMClient object. Returns None otherwise.
"""
subdomain = get_subdomain(request)
scim_config_dict = settings.SCIM_CONFIG.get(subdomain)
if not scim_config_dict:
return None
valid_bearer_token = scim_config_dict.get("bearer_token")
scim_client_name = scim_config_dict.get("scim_client_name")
# We really don't want a misconfiguration where this is unset,
# allowing free access to the SCIM API:
assert valid_bearer_token
assert scim_client_name
if request.headers.get("Authorization") != f"Bearer {valid_bearer_token}":
return None
request_notes = RequestNotes.get_notes(request)
assert request_notes.realm
# While API authentication code paths are sufficiently high
# traffic that we prefer to use a cache, SCIM is much lower
# traffic, and doing a database query is plenty fast.
return SCIMClient.objects.get(realm=request_notes.realm, name=scim_client_name)
def process_request(self, request: HttpRequest) -> Optional[HttpResponse]:
# Match against ALLOWED_HOSTS, which is rather permissive;
# failure will raise DisallowedHost, which is a 400.
request.get_host()
# This check is important to avoid doing the extra work of
# `get_realm` (which does a database query that could be
# problematic for Tornado). Also the error page below is only
# appropriate for a page visited in a browser, not the API.
#
# API authentication will end up checking for an invalid
# realm, and throw a JSON-format error if appropriate.
if request.path.startswith(("/static/", "/api/", "/json/")):
return None
subdomain = get_subdomain(request)
if (
subdomain != Realm.SUBDOMAIN_FOR_ROOT_DOMAIN
and subdomain != settings.SOCIAL_AUTH_SUBDOMAIN
):
request_notes = RequestNotes.get_notes(request)
try:
request_notes.realm = get_realm(subdomain)
except Realm.DoesNotExist:
return render(request, "zerver/invalid_realm.html", status=404)
request_notes.has_fetched_realm = True
return None
def process_response(
self, request: HttpRequest, response: HttpResponseBase
) -> HttpResponseBase:
# This is the same as the default LocaleMiddleware, minus the
# logic that redirects 404's that lack a prefixed language in
# the path into having a language. See
# https://code.djangoproject.com/ticket/32005
language = translation.get_language()
language_from_path = translation.get_language_from_path(request.path_info)
urlconf = getattr(request, "urlconf", settings.ROOT_URLCONF)
i18n_patterns_used, _ = is_language_prefix_patterns_used(urlconf)
if not (i18n_patterns_used and language_from_path):
patch_vary_headers(response, ("Accept-Language",))
assert language is not None
response.setdefault("Content-Language", language)
# An additional responsibility of our override of this middleware is to save the user's language
# preference in a cookie. That determination is made by code handling the request
# and saved in the set_language flag so that it can be used here.
set_language = RequestNotes.get_notes(request).set_language
if set_language is not None:
response.set_cookie(settings.LANGUAGE_COOKIE_NAME, set_language)
return response
def process_response(
self, request: HttpRequest, response: StreamingHttpResponse
) -> StreamingHttpResponse:
if RequestNotes.get_notes(request).placeholder_open_graph_description is not None:
assert not response.streaming
response.content = alter_content(request, response.content)
return response
def async_request_timer_restart(request: HttpRequest) -> None:
log_data = RequestNotes.get_notes(request).log_data
assert log_data is not None
if "time_restarted" in log_data:
# Don't destroy data when being called from
# finish_current_handler
return
record_request_restart_data(log_data)
def client_is_exempt_from_rate_limiting(request: HttpRequest) -> bool:
# Don't rate limit requests from Django that come from our own servers,
# and don't rate-limit dev instances
client = RequestNotes.get_notes(request).client
return (client is not None
and client.name.lower() == "internal") and (is_local_addr(
request.META["REMOTE_ADDR"]) or settings.DEBUG_RATE_LIMITING)
def get_realm_from_request(request: HttpRequest) -> Optional[Realm]:
request_notes = RequestNotes.get_notes(request)
if hasattr(request, "user") and hasattr(request.user, "realm"):
return request.user.realm
if not request_notes.has_fetched_realm:
# We cache the realm object from this function on the request data,
# so that functions that call get_realm_from_request don't
# need to do duplicate queries on the same realm while
# processing a single request.
subdomain = get_subdomain(request)
request_notes = RequestNotes.get_notes(request)
try:
request_notes.realm = get_realm(subdomain)
except Realm.DoesNotExist:
request_notes.realm = None
request_notes.has_fetched_realm = True
return request_notes.realm
def __init__(
self,
post_data: Dict[str, Any] = {},
user_profile: Optional[Union[UserProfile, AnonymousUser,
RemoteZulipServer]] = None,
host: str = settings.EXTERNAL_HOST,
client_name: Optional[str] = None,
meta_data: Optional[Dict[str, Any]] = None,
tornado_handler: Optional[AsyncDjangoHandler] = DummyHandler(),
path: str = "",
) -> None:
self.host = host
self.GET = QueryDict(mutable=True)
self.method = ""
# Convert any integer parameters passed into strings, even
# though of course the HTTP API would do so. Ideally, we'd
# get rid of this abstraction entirely and just use the HTTP
# API directly, but while it exists, we need this code
self.POST = QueryDict(mutable=True)
for key in post_data:
self.POST[key] = str(post_data[key])
self.method = "POST"
if meta_data is None:
self.META = {"PATH_INFO": "test"}
else:
self.META = meta_data
self.path = path
self.user = user_profile
self._body = b""
self.content_type = ""
BaseNotes[str, str].get_notes
RequestNotes.set_notes(
self,
RequestNotes(
client_name="",
log_data={},
tornado_handler=None
if tornado_handler is None else weakref.ref(tornado_handler),
client=get_client(client_name)
if client_name is not None else None,
),
)
def alter_content(request: HttpRequest, content: bytes) -> bytes:
first_paragraph_text = get_content_description(content, request)
placeholder_open_graph_description = RequestNotes.get_notes(
request).placeholder_open_graph_description
assert placeholder_open_graph_description is not None
return content.replace(
placeholder_open_graph_description.encode(),
first_paragraph_text.encode(),
)
def mark_all_as_read(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
request_notes = RequestNotes.get_notes(request)
count = do_mark_all_as_read(user_profile)
log_data_str = f"[{count} updated]"
assert request_notes.log_data is not None
request_notes.log_data["extra"] = log_data_str
return json_success(request)
def do_login(request: HttpRequest, user_profile: UserProfile) -> None:
"""Creates a session, logging in the user, using the Django method,
and also adds helpful data needed by our server logs.
"""
django_login(request, user_profile)
RequestNotes.get_notes(request).requestor_for_logs = user_profile.format_requestor_for_logs()
process_client(request, user_profile, is_browser_view=True)
if settings.TWO_FACTOR_AUTHENTICATION_ENABLED:
# Log in with two factor authentication as well.
do_two_factor_login(request, user_profile)
def mark_all_as_read(request: HttpRequest, user_profile: UserProfile) -> HttpResponse:
request_notes = RequestNotes.get_notes(request)
assert request_notes.client is not None
count = do_mark_all_as_read(user_profile, request_notes.client)
log_data_str = f"[{count} updated]"
assert request_notes.log_data is not None
request_notes.log_data["extra"] = log_data_str
return json_success({"result": "success", "msg": ""})
def process_response(self, request: HttpRequest, response: HttpResponse) -> HttpResponse:
if not settings.RATE_LIMITING:
return response
# Add X-RateLimit-*** headers
ratelimits_applied = RequestNotes.get_notes(request).ratelimits_applied
if len(ratelimits_applied) > 0:
self.set_response_headers(response, ratelimits_applied)
return response
def base_scim_location_getter(request: HttpRequest, *args: Any, **kwargs: Any) -> str:
"""Used as the base url for constructing the Location of a SCIM resource.
Since SCIM synchronization is scoped to an individual realm, we
need these locations to be namespaced within the realm's domain
namespace, which is conveniently accessed via realm.uri.
"""
realm = RequestNotes.get_notes(request).realm
assert realm is not None
return realm.uri
def mark_stream_as_read(
request: HttpRequest, user_profile: UserProfile, stream_id: int = REQ(json_validator=check_int)
) -> HttpResponse:
stream, sub = access_stream_by_id(user_profile, stream_id)
count = do_mark_stream_messages_as_read(user_profile, stream.recipient_id)
log_data_str = f"[{count} updated]"
log_data = RequestNotes.get_notes(request).log_data
assert log_data is not None
log_data["extra"] = log_data_str
return json_success({"result": "success", "msg": ""})
def render_message_backend(
request: HttpRequest, user_profile: UserProfile, content: str = REQ()
) -> HttpResponse:
message = Message()
message.sender = user_profile
message.content = content
client = RequestNotes.get_notes(request).client
assert client is not None
message.sending_client = client
rendering_result = render_markdown(message, content, realm=user_profile.realm)
return json_success(request, data={"rendered": rendering_result.rendered_content})
def api_fetch_api_key(
request: HttpRequest, username: str = REQ(), password: str = REQ()
) -> HttpResponse:
return_data: Dict[str, bool] = {}
realm = get_realm_from_request(request)
if realm is None:
raise InvalidSubdomainError()
if not ldap_auth_enabled(realm=realm):
# In case we don't authenticate against LDAP, check for a valid
# email. LDAP backend can authenticate against a non-email.
validate_login_email(username)
user_profile = authenticate(request=request,
username=username,
password=password,
realm=realm,
return_data=return_data)
if return_data.get("inactive_user"):
raise UserDeactivatedError()
if return_data.get("inactive_realm"):
raise RealmDeactivatedError()
if return_data.get("password_auth_disabled"):
raise PasswordAuthDisabledError()
if return_data.get("password_reset_needed"):
raise PasswordResetRequiredError()
if user_profile is None:
raise AuthenticationFailedError()
assert user_profile.is_authenticated
# Maybe sending 'user_logged_in' signal is the better approach:
# user_logged_in.send(sender=user_profile.__class__, request=request, user=user_profile)
# Not doing this only because over here we don't add the user information
# in the session. If the signal receiver assumes that we do then that
# would cause problems.
email_on_new_login(sender=user_profile.__class__,
request=request,
user=user_profile)
# Mark this request as having a logged-in user for our server logs.
assert isinstance(user_profile, UserProfile)
process_client(request, user_profile)
RequestNotes.get_notes(
request).requestor_for_logs = user_profile.format_requestor_for_logs()
api_key = get_api_key(user_profile)
return json_success(request,
data={
"api_key": api_key,
"email": user_profile.delivery_email
})
def cleanup_event_queue(
request: HttpRequest, user_profile: UserProfile, queue_id: str = REQ()
) -> HttpResponse:
client = get_client_descriptor(str(queue_id))
if client is None:
raise BadEventQueueIdError(queue_id)
if user_profile.id != client.user_profile_id:
raise JsonableError(_("You are not authorized to access this queue"))
log_data = RequestNotes.get_notes(request).log_data
assert log_data is not None
log_data["extra"] = f"[{queue_id}]"
in_tornado_thread(client.cleanup)
return json_success(request)
def report_unnarrow_times(
request: HttpRequest,
user_profile: Union[UserProfile, AnonymousUser],
initial_core: int = REQ(converter=to_non_negative_int),
initial_free: int = REQ(converter=to_non_negative_int),
) -> HttpResponse:
log_data = RequestNotes.get_notes(request).log_data
assert log_data is not None
log_data["extra"] = f"[{initial_core}ms/{initial_free}ms]"
realm = get_valid_realm_from_request(request)
base_key = statsd_key(realm.string_id, clean_periods=True)
statsd.timing(f"unnarrow.initial_core.{base_key}", initial_core)
statsd.timing(f"unnarrow.initial_free.{base_key}", initial_free)
return json_success(request)
