def combinedLogFormatter(timestamp: str, request: IRequest) -> str:
"""
@return: A combined log formatted log line for the given request.
@see: L{IAccessLogFormatter}
"""
referrer = _escape(request.getHeader("referer") or "-")
agent = _escape(request.getHeader("user-agent") or "-")
clientIP = request.getClientIP()
forwardedFor = (
request.requestHeaders
.getRawHeaders(b"x-forwarded-for", [b""])[0]
.split(b",")[0]
.strip()
).decode("charmap")
if forwardedFor:
ip = f"{forwardedFor} > {clientIP}"
else:
ip = clientIP
if hasattr(request, "user") and request.user is not None:
username = request.user.shortNames[0]
try:
username = _escape(username)
except Exception:
username = _escape(repr(username))
else:
username = "******"
line = (
'"{ip}" {user} - {timestamp} "{method} {uri} {protocol}" '
'{code} {length} "{referrer}" "{agent}"'.format(
ip=_escape(ip or "-"),
timestamp=timestamp,
method=_escape(request.method),
uri=_escape(request.uri),
protocol=_escape(request.clientproto),
code=request.code,
length=request.sentLength or "-",
referrer=referrer,
agent=agent,
user=username,
)
)
return line
def urlFor(
self,
request: IRequest,
endpoint: str,
values: Optional[Mapping[str, KleinQueryValue]] = None,
method: Optional[str] = None,
force_external: bool = False,
append_unknown: bool = True,
) -> str:
host = request.getHeader(b"host")
if host is None:
if force_external:
raise ValueError(
"Cannot build external URL if request"
" doesn't contain Host header"
)
host = b""
return buildURL(
self.url_map.bind(host),
endpoint,
values,
method,
force_external,
append_unknown,
)
def extractValue(self, request: IRequest) -> Any:
"""
Extract a value from the request.
In the case of key/value form posts, this attempts to reliably make the
value into str. In the case of a JSON post, however, it will simply
extract the value from the top-level dictionary, which means it could
be any arrangement of JSON-serializiable objects.
"""
fieldName = self.formFieldName
if fieldName is None:
raise ValueError("Cannot extract unnamed form field.")
contentType = request.getHeader(b"content-type")
if contentType is not None and contentType.startswith(
b"application/json"):
# TODO: parse only once, please.
parsed = request.getComponent(IParsedJSONBody)
if parsed is None:
request.content.seek(0)
octets = request.content.read()
characters = octets.decode("utf-8")
parsed = json.loads(characters)
request.setComponent(IParsedJSONBody, parsed)
if fieldName not in parsed:
return None
return parsed[fieldName]
allValues = request.args.get(fieldName.encode("utf-8"))
if allValues:
return allValues[0].decode("utf-8")
else:
return None
def combinedLogFormatter(timestamp: str, request: IRequest) -> str:
"""
@return: A combined log formatted log line for the given request.
@see: L{IAccessLogFormatter}
"""
referrer = _escape(request.getHeader("referer") or "-")
agent = _escape(request.getHeader("user-agent") or "-")
clientIP = request.getClientIP()
forwardedFor = (
request.requestHeaders.getRawHeaders(b"x-forwarded-for", [b""])[0]
.split(b",")[0]
.strip()
).decode("charmap")
if forwardedFor:
ip = f"{forwardedFor} > {clientIP}"
else:
ip = clientIP
if hasattr(request, "user") and request.user is not None:
username = request.user.shortNames[0]
try:
username = _escape(username)
except Exception:
username = _escape(repr(username))
else:
username = "******"
line = (
'"{ip}" {user} - {timestamp} "{method} {uri} {protocol}" '
'{code} {length} "{referrer}" "{agent}"'.format(
ip=_escape(ip or "-"),
timestamp=timestamp,
method=_escape(request.method),
uri=_escape(request.uri),
protocol=_escape(request.clientproto),
code=request.code,
length=request.sentLength or "-",
referrer=referrer,
agent=agent,
user=username,
)
)
return line
def render(self, request: IRequest) -> bytes:
# Pick an encoding based on client and server preferences.
# We strongly prefer gzip because we save on bandwidth and
# have pre-compressed the resource.
accept = AcceptedEncodings.parse(request.getHeader('accept-encoding'))
if 4.0 * accept['gzip'] > accept['identity']:
request.setHeader('Content-Encoding', 'gzip')
return self.__gzippedResource.render(request)
else:
return super().render(request)
def get_request_user_agent(request: IRequest, default: str = "") -> str:
"""Return the last User-Agent header, or the given default."""
# There could be raw utf-8 bytes in the User-Agent header.
# N.B. if you don't do this, the logger explodes cryptically
# with maximum recursion trying to log errors about
# the charset problem.
# c.f. https://github.com/matrix-org/synapse/issues/3471
h = request.getHeader(b"User-Agent")
return h.decode("ascii", "replace") if h else default
def __init__(self, request: IRequest, frameAncestors: str,
userAgent: UserAgent):
super().__init__()
self._request = request
self._frameAncestors = frameAncestors
self.userAgent = userAgent
# Determine whether or not we will gzip the body.
# We prefer gzip to save on bandwidth.
accept = AcceptedEncodings.parse(request.getHeader('accept-encoding'))
self._gzipContent = 2.0 * accept['gzip'] > accept['identity']
def notAuthenticatedError(app: Any, request: IRequest,
failure: Failure) -> KleinRenderable:
"""
Not authenticated.
"""
requestedWith = request.getHeader("X-Requested-With")
if requestedWith is not None:
if requestedWith == "XMLHttpRequest":
return forbiddenResponse(request)
element = redirect(request, URLs.login, origin="o")
return renderElement(request, element)
def notAuthenticatedError(
app: Any, request: IRequest, failure: Failure
) -> KleinRenderable:
"""
Not authenticated.
"""
requestedWith = request.getHeader("X-Requested-With")
if requestedWith is not None:
if requestedWith == "XMLHttpRequest":
return forbiddenResponse(request)
element = redirect(request, URLs.login, origin="o")
return renderElement(request, element)
def defaultValidationFailureHandler(
instance: Optional[object],
request: IRequest,
fieldValues: "FieldValues",
) -> Element:
"""
This is the default validation failure handler, which will be used by form
handlers (i.e. any routes which use L{klein.Requirer} to require a field)
in the case of any input validation failure when no other validation
failure handler is registered via L{Form.onValidationFailureFor}.
Its behavior is to simply return an HTML rendering of the form object,
which includes inline information about fields which failed to validate.
@param instance: The instance associated with the router that the form
handler was handled on.
@type instance: L{object}
@param request: The request including the form submission.
@type request: L{twisted.web.iweb.IRequest}
@return: Any object acceptable from a Klein route.
"""
session = request.getComponent(ISession)
request.setResponseCode(400)
enctype = ((request.getHeader(b"content-type")
or RenderableForm.ENCTYPE_URL_ENCODED.encode("ascii")
).split(b";")[0].decode("charmap"))
renderable = RenderableForm(
fieldValues.form,
session,
"/".join(
segment.decode("utf-8", errors="replace")
for segment in request.prepath),
request.method,
enctype,
"utf-8",
fieldValues.prevalidationValues,
fieldValues.validationErrors,
)
return Element(TagLoader(renderable))
def urlFromRequest(request: IRequest) -> DecodedURL:
sentHeader = request.getHeader(b"host")
if sentHeader is not None:
sentHeader = sentHeader.decode("charmap")
if ":" in sentHeader:
host, port = sentHeader.split(":")
port = int(port)
else:
host = sentHeader
port = None
else:
host = request.client.host
port = request.client.port
url = DecodedURL.fromText(request.uri.decode("charmap"))
url = url.replace(
scheme="https" if request.isSecure() else "http",
host=host,
port=port,
)
return url
def _get_requested_host(request: IRequest) -> bytes:
hostname = request.getHeader(b"host")
if hostname:
return hostname
# no Host header, use the address/port that the request arrived on
host: Union[address.IPv4Address, address.IPv6Address] = request.getHost()
hostname = host.host.encode("ascii")
if request.isSecure() and host.port == 443:
# default port for https
return hostname
if not request.isSecure() and host.port == 80:
# default port for http
return hostname
return b"%s:%i" % (
hostname,
host.port,
)
def procureSession(self,
request: IRequest,
forceInsecure: bool = False) -> Any:
alreadyProcured = request.getComponent(ISession)
if alreadyProcured is not None:
if not forceInsecure or not request.isSecure():
returnValue(alreadyProcured)
if request.isSecure():
if forceInsecure:
tokenHeader = self._insecureTokenHeader
cookieName: Union[str, bytes] = self._insecureCookie
sentSecurely = False
else:
tokenHeader = self._secureTokenHeader
cookieName = self._secureCookie
sentSecurely = True
else:
# Have we inadvertently disclosed a secure token over an insecure
# transport, for example, due to a buggy client?
allPossibleSentTokens: Sequence[str] = sum(
[
request.requestHeaders.getRawHeaders(header, [])
for header in [
self._secureTokenHeader,
self._insecureTokenHeader,
]
],
[],
) + [
it for it in [
request.getCookie(cookie)
for cookie in [self._secureCookie, self._insecureCookie]
] if it
]
# Does it seem like this check is expensive? It sure is! Don't want
# to do it? Turn on your dang HTTPS!
yield self._store.sentInsecurely(allPossibleSentTokens)
tokenHeader = self._insecureTokenHeader
cookieName = self._insecureCookie
sentSecurely = False
# Fun future feature: honeypot that does this over HTTPS, but sets
# isSecure() to return false because it serves up a cert for the
# wrong hostname or an invalid cert, to keep API clients honest
# about chain validation.
sentHeader = (request.getHeader(tokenHeader) or b"").decode("utf-8")
sentCookie = (request.getCookie(cookieName) or b"").decode("utf-8")
if sentHeader:
mechanism = SessionMechanism.Header
else:
mechanism = SessionMechanism.Cookie
if not (sentHeader or sentCookie):
session = None
else:
try:
session = yield self._store.loadSession(
sentHeader or sentCookie, sentSecurely, mechanism)
except NoSuchSession:
if mechanism == SessionMechanism.Header:
raise
session = None
if mechanism == SessionMechanism.Cookie and (
session is None or session.identifier != sentCookie):
if session is None:
if request.startedWriting:
# At this point, if the mechanism is Header, we either have
# a valid session or we bailed after NoSuchSession above.
raise TooLateForCookies(
"You tried initializing a cookie-based session too"
" late in the request pipeline; the headers"
" were already sent.")
if request.method != b"GET":
# Sessions should only ever be auto-created by GET
# requests; there's no way that any meaningful data
# manipulation could succeed (no CSRF token check could
# ever succeed, for example).
raise NoSuchSession(
"Can't initialize a session on a "
"{method} request.".format(
method=request.method.decode("ascii")))
if not self._setCookieOnGET:
# We don't have a session ID at all, and we're not allowed
# by policy to set a cookie on the client.
raise NoSuchSession(
"Cannot auto-initialize a session for this request.")
session = yield self._store.newSession(sentSecurely, mechanism)
identifierInCookie = session.identifier
if not isinstance(identifierInCookie, str):
identifierInCookie = identifierInCookie.encode("ascii")
if not isinstance(cookieName, str):
cookieName = cookieName.decode("ascii")
request.addCookie(
cookieName,
identifierInCookie,
max_age=str(self._maxAge),
domain=self._cookieDomain,
path=self._cookiePath,
secure=sentSecurely,
httpOnly=True,
)
if sentSecurely or not request.isSecure():
# Do not cache the insecure session on the secure request, thanks.
request.setComponent(ISession, session)
returnValue(session)
