def unicode_body(self):
if not okay(self.decoded_body):
return self.decoded_body
if not okay(self.guessed_charset):
return Unavailable(self.decoded_body)
# pylint: disable=no-member
return self.decoded_body.decode(self.guessed_charset)
def _check_day_of_week(complain, r):
(claimed_dow, r) = r
if okay(r) and r.weekday() != claimed_dow:
complain(1108, date=r.strftime('%Y-%m-%d'),
claimed=_DAY_NAMES[claimed_dow],
actual=_DAY_NAMES[r.weekday()])
return r
def _parse_response_body(resp, stream):
req = resp.request
# RFC 7230 section 3.3.3.
if resp.status == st.switching_protocols:
resp.body = b''
resp.complain(1011)
stream.sane = False
elif req and req.method == m.CONNECT and resp.status.successful:
resp.body = b''
resp.complain(1012)
stream.sane = False
elif (resp.status.informational or
resp.status in [st.no_content, st.not_modified] or
(req and req.method == m.HEAD)):
resp.body = b''
elif resp.headers.transfer_encoding:
codings = resp.headers.transfer_encoding.value[:]
if codings[-1] == tc.chunked:
codings.pop()
_parse_chunked(resp, stream)
else:
resp.body = stream.read()
while codings and okay(resp.body):
_decode_transfer_coding(resp, codings.pop())
elif resp.headers.content_length.is_present:
_process_content_length(resp, stream)
else:
resp.body = stream.read()
def _check_multipart_byteranges(resp):
if okay(resp.multipart_data):
for i, part in enumerate(resp.multipart_data.get_payload()):
if u'Content-Range' not in part:
resp.complain(1141, part_num=(i + 1))
if u'Content-Type' not in part:
resp.complain(1142, part_num=(i + 1))
def displayable_body(self):
"""
The payload body in a form that is appropriate for display in a message
preview, along with a list of phrases explaining which transformations
have been applied to arrive at that form.
"""
removing_te = [u'removing Transfer-Encoding'] \
if self.headers.transfer_encoding else []
removing_ce = [u'removing Content-Encoding'] \
if self.headers.content_encoding else []
decoding_charset = [u'decoding from %s' % self.guessed_charset] \
if self.guessed_charset and self.guessed_charset != 'utf-8' else []
pretty_printing = [u'pretty-printing']
if okay(self.json_data):
r = json.dumps(self.json_data, indent=2, ensure_ascii=False)
transforms = \
removing_te + removing_ce + decoding_charset + pretty_printing
elif okay(self.unicode_body):
r = self.unicode_body
transforms = removing_te + removing_ce + decoding_charset
elif okay(self.decoded_body):
# pylint: disable=no-member
r = self.decoded_body.decode('utf-8', 'replace')
transforms = removing_te + removing_ce
elif okay(self.body):
r = self.body.decode('utf-8', 'replace')
transforms = removing_te
else:
return self.body, []
limit = 1000
if len(r) > limit:
r = r[:limit]
transforms += [u'taking the first %d characters' % limit]
pr = printable(r)
if r != pr:
r = pr
transforms += [u'replacing non-printable characters '
u'with the \ufffd sign']
return r, transforms
def __init__(self, version, header_entries, body, trailer_entries=None):
super(Message, self).__init__()
self.version = (HTTPVersion(force_unicode(version))
if version is not None else None)
self.header_entries = [HeaderEntry(k, v)
for k, v in header_entries]
self.body = bytes(body) if okay(body) else body
self.trailer_entries = [HeaderEntry(k, v)
for k, v in trailer_entries or []]
self.rebuild_headers()
self.annotations = {}
def json_data(self):
if self.headers.content_type.is_okay and \
media_type.is_json(self.headers.content_type.value.item) and \
okay(self.unicode_body) and self.content_is_full:
try:
return json.loads(self.unicode_body)
except ValueError as e:
self.complain(1038, error=e)
return Unavailable
else:
return None
def url_encoded_data(self):
if self.headers.content_type == \
media.application_x_www_form_urlencoded and \
okay(self.decoded_body) and self.content_is_full:
for char in iterbytes(self.decoded_body):
if not URL_ENCODED_GOOD_CHARS[ord(char)]:
self.complain(1040, char=format_chars([char]))
return Unavailable(self.decoded_body)
# pylint: disable=no-member
return parse_qs(self.decoded_body.decode('ascii'))
return None
def content_is_full(self):
"""Does this response carry a complete instance of its Content-Type?"""
if self.status in [st.not_modified, st.early_hints]:
return False
if self.status == st.partial_content and \
self.headers.content_type != media.multipart_byteranges:
return False
if okay(self.request):
return self.request.method != m.HEAD
if self.body:
return True
return None # pragma: no cover
def decoded_body(self):
"""The payload body with Content-Encoding removed."""
r = self.body
codings = self.headers.content_encoding.value[:]
while codings and okay(r) and r:
coding = codings.pop()
decoder = {cc.gzip: decode_gzip,
cc.x_gzip: decode_gzip,
cc.deflate: decode_deflate,
cc.br: decode_brotli}.get(coding)
if decoder is not None:
try:
r = decoder(r)
except Exception as e:
self.complain(1037, coding=coding, error=e)
r = Unavailable(r)
elif okay(coding):
self.complain(1036, coding=coding)
r = Unavailable(r)
else:
r = Unavailable(r)
return r
def test_tcpick_request_timeout():
[box, exch1] = load_from_tcpick('request_timeout')
# The first exchange is only a box for no. 1278.
assert box.request is None
assert box.responses == []
assert [complaint.id for complaint in box.complaints] == [1278]
# The second exchange contains only the 408 response.
assert exch1.request is None
[resp] = exch1.responses
assert resp.status == st.request_timeout
assert okay(resp.xml_data)
def url_encoded_data(self):
if self.headers.content_type == \
media.application_x_www_form_urlencoded and \
okay(self.decoded_body) and self.content_is_full:
for byte in six.iterbytes(self.decoded_body):
if not URL_ENCODED_GOOD_BYTES[byte]:
char = six.int2byte(byte)
self.complain(1040, char=format_chars([char]))
return Unavailable
# pylint: disable=no-member
return parse_qs(self.decoded_body.decode('ascii'))
else:
return None
def decoded_body(self):
r = self.body
codings = list(self.headers.content_encoding)
while codings and okay(r) and r:
coding = codings.pop()
if coding in [cc.gzip, cc.x_gzip]:
try:
r = decode_gzip(r)
except Exception as e:
self.complain(1037, coding=coding, error=e)
r = Unavailable
elif coding == cc.deflate:
try:
r = decode_deflate(r)
except Exception as e:
self.complain(1037, coding=coding, error=e)
r = Unavailable
elif okay(coding):
self.complain(1036, coding=coding)
r = Unavailable
else:
r = Unavailable
return r
def json_data(self):
if self.headers.content_type.is_okay and \
known.media_type.is_json(self.headers.content_type.item) and \
okay(self.unicode_body) and self.content_is_full:
try:
r = json.loads(self.unicode_body)
except ValueError as e:
self.complain(1038, error=e)
r = Unavailable(self.unicode_body)
else:
if self.guessed_charset not in ['ascii', 'utf-8', None]:
self.complain(1281)
return r
return None
def _displayable_body(msg):
removing_te = [u'removing Transfer-Encoding'] \
if msg.headers.transfer_encoding else []
removing_ce = [u'removing Content-Encoding'] \
if msg.headers.content_encoding else []
decoding_charset = [u'decoding from %s' % msg.guessed_charset] \
if msg.guessed_charset and msg.guessed_charset != 'utf-8' else []
pretty_printing = [u'pretty-printing']
if okay(msg.json_data):
r = json.dumps(msg.json_data, indent=2, ensure_ascii=False)
transforms = \
removing_te + removing_ce + decoding_charset + pretty_printing
elif okay(msg.unicode_body):
r = msg.unicode_body
transforms = removing_te + removing_ce + decoding_charset
elif okay(msg.decoded_body):
r = msg.decoded_body.decode('utf-8', 'replace')
transforms = removing_te + removing_ce
elif okay(msg.body):
r = msg.body.decode('utf-8', 'replace')
transforms = removing_te
else:
return msg.body, []
limit = 1000
if len(r) > limit:
r = r[:limit]
transforms += [u'taking the first %d characters' % limit]
pr = printable(r)
if r != pr:
r = pr
transforms += [u'replacing non-printable characters '
u'with the \ufffd sign']
return r, transforms
def _process_content_length(msg, stream):
n = msg.headers.content_length.value
if not okay(n):
msg.body = Unavailable()
stream.sane = False
elif n > MAX_BODY_SIZE:
msg.body = Unavailable()
stream.sane = False
msg.complain(1298, place=msg.headers.content_length, size=n,
max_size=MAX_BODY_SIZE)
else:
try:
msg.body = stream.read(n)
except ParseError as exc:
msg.body = Unavailable()
msg.complain(1004, error=exc)
def xml_data(self):
if self.headers.content_type.is_okay and \
media_type.is_xml(self.headers.content_type.value.item) and \
okay(self.decoded_body) and self.content_is_full:
try:
# It's not inconceivable that a message might contain
# maliciously constructed XML data, so we use `defusedxml`.
return defusedxml.ElementTree.fromstring(self.decoded_body)
except defusedxml.EntitiesForbidden:
self.complain(1275)
return Unavailable
except xml.etree.ElementTree.ParseError as e:
self.complain(1039, error=e)
return Unavailable
else:
return None
def delimited_by_close(self):
if self.headers.content_length.is_present or \
tc.chunked in self.headers.transfer_encoding or \
self.status.informational or \
self.status in [st.no_content, st.not_modified] or \
self.version == http2:
return False
if okay(self.request):
if self.request.method == m.HEAD:
return False
if self.request.method == m.CONNECT and self.status.successful:
return False
if self.version in [http10, http11]:
self.complain(1025)
return True
return None
def xml_data(self):
if self.headers.content_type.is_okay and \
known.media_type.is_xml(self.headers.content_type.item) and \
okay(self.decoded_body) and self.content_is_full:
try:
# It's not inconceivable that a message might contain
# maliciously constructed XML data, so we use `defusedxml`.
return defusedxml.ElementTree.fromstring(self.decoded_body)
except defusedxml.EntitiesForbidden:
self.complain(1275)
return Unavailable(self.decoded_body)
# https://bugs.python.org/issue29896
except (xml.etree.ElementTree.ParseError, UnicodeError) as e:
self.complain(1039, error=e)
return Unavailable(self.decoded_body)
else:
return None
def guessed_charset(self):
charset = 'utf-8'
if self.headers.content_type.is_okay:
charset = self.headers.content_type.param.get(u'charset', charset)
try:
codec = codecs.lookup(charset)
except (LookupError, UnicodeError):
return None
charset = codec.name
if okay(self.decoded_body):
try:
self.decoded_body.decode(charset) # pylint: disable=no-member
except UnicodeError:
return None
return charset
def multipart_data(self):
ctype = self.headers.content_type
if ctype.is_okay and media_type.is_multipart(ctype.value.item) and \
okay(self.decoded_body) and self.content_is_full:
# All multipart media types obey the same general syntax
# specified in RFC 2046 Section 5.1,
# and should be parseable as email message payloads.
multipart_code = (b'Content-Type: ' + ctype.entries[0].value +
b'\r\n\r\n' + self.decoded_body)
parsed = parse_email_message(multipart_code)
for d in parsed.defects:
if isinstance(d, email.errors.NoBoundaryInMultipartDefect):
self.complain(1139)
elif isinstance(d, email.errors.StartBoundaryNotFoundDefect):
self.complain(1140)
return parsed if parsed.is_multipart() else Unavailable
else:
return None
def _parse_response_body(resp, stream):
req = resp.request
# RFC 7230 section 3.3.3.
if resp.status == st.switching_protocols:
resp.body = b''
resp.complain(1011)
stream.sane = False
elif req and req.method == m.CONNECT and resp.status.successful:
resp.body = b''
resp.complain(1012)
stream.sane = False
elif (resp.status.informational or
resp.status in [st.no_content, st.not_modified] or
(req and req.method == m.HEAD)):
resp.body = b''
elif resp.headers.transfer_encoding:
codings = list(resp.headers.transfer_encoding)
if codings[-1] == tc.chunked:
codings.pop()
_parse_chunked(resp, stream)
else:
resp.body = stream.consume_rest()
while codings and okay(resp.body):
_decode_transfer_coding(resp, codings.pop())
elif resp.headers.content_length.is_present:
n = resp.headers.content_length.value
if n is Unavailable:
resp.body = Unavailable
stream.sane = False
else:
try:
resp.body = stream.consume_n_bytes(n)
except ParseError:
resp.body = Unavailable
resp.complain(1004)
stream.sane = False
else:
resp.body = stream.consume_rest()
def _parse_request_body(req, stream):
# RFC 7230 section 3.3.3.
if req.headers.transfer_encoding:
codings = req.headers.transfer_encoding.value[:]
if codings.pop() == tc.chunked:
_parse_chunked(req, stream)
else:
req.body = Unavailable()
req.complain(1001)
stream.sane = False
while codings and okay(req.body):
_decode_transfer_coding(req, codings.pop())
elif req.headers.content_length:
_process_content_length(req, stream)
else:
req.body = b''
def _decode_transfer_coding(msg, coding):
if coding == tc.chunked:
# The outermost chunked has already been peeled off at this point.
msg.complain(1002)
msg.body = Unavailable(msg.body)
elif coding in [tc.gzip, tc.x_gzip]:
try:
msg.body = decode_gzip(msg.body)
except Exception as e:
msg.complain(1027, coding=coding, error=e)
msg.body = Unavailable(msg.body)
elif coding == tc.deflate:
try:
msg.body = decode_deflate(msg.body)
except Exception as e:
msg.complain(1027, coding=coding, error=e)
msg.body = Unavailable(msg.body)
else:
if okay(coding):
msg.complain(1003, coding=coding)
msg.body = Unavailable(msg.body)
def target_form(self):
try:
target = self.target.encode('iso-8859-1')
except UnicodeError as e:
self.complain(1045, error=e)
return None
if self.method == m.CONNECT:
parser = mark(authority_form)
elif self.method == m.OPTIONS:
parser = (mark(origin_form) | mark(asterisk_form) |
mark(absolute_form))
else:
parser = mark(origin_form) | mark(absolute_form)
r = simple_parse(target, parser,
self.complain, 1045, place=u'request target')
if okay(r):
(symbol, _) = r
return symbol
else:
return None
def _compare(self, other, op):
# It would be nice to be able to compare headers by values, as in::
#
# resp.headers.last_modified == req.headers.if_modified_since
#
# Unfortunately, there are places
# (such as :meth:`httpolice.blackboard.Blackboard.complain`)
# where we need header-to-header equality to be less magical.
# And if we can't do this magic for equality,
# there's no sense in doing it for other operations.
# So we just say that comparing headers to headers is `NotImplemented`
# (fall back to comparing their object identities).
#
# Now, the following form still works::
#
# resp.headers.last_modified == req.headers.if_modified_since.value
#
# so we don't lose all that much.
if isinstance(other, HeaderView):
return NotImplemented
return self.is_okay and okay(other) and op(self.value, other)
def check_message(msg):
"""Run all checks that apply to any message (both request and response)."""
complain = msg.complain
version = msg.version
headers = msg.headers
x_prefixed = []
for hdr in headers:
# Check the header name syntax.
parse(hdr.name,
rfc7230.field_name,
complain,
1293,
header=hdr,
place=u'field name')
# Force parsing every header present in the message
# according to its syntax rules.
_ = hdr.value
if known.header.is_deprecated(hdr.name):
complain(1197, header=hdr)
if hdr.name.startswith(u'X-') and hdr.name not in known.header:
x_prefixed.append(hdr)
if x_prefixed:
complain(1277, headers=x_prefixed)
# Force checking the payload according to various rules.
_ = msg.decoded_body
_ = msg.unicode_body
_ = msg.json_data
_ = msg.xml_data
_ = msg.multipart_data
_ = msg.url_encoded_data
if version == http11 and headers.trailer.is_present and \
tc.chunked not in headers.transfer_encoding:
# HTTP/2 supports trailers but has no notion of "chunked".
complain(1054)
for entry in msg.trailer_entries:
if entry.name not in headers.trailer:
complain(1030, header=entry)
if headers.transfer_encoding.is_present and \
headers.content_length.is_present:
complain(1020)
for opt in headers.connection:
if known.header.is_bad_for_connection(FieldName(opt)):
complain(1034, header=headers[FieldName(opt)])
if headers.content_type.is_okay:
if known.media_type.is_deprecated(headers.content_type.item):
complain(1035)
for dupe in headers.content_type.param.duplicates():
complain(1042, param=dupe)
if headers.content_type == media.application_json and \
u'charset' in headers.content_type.param:
complain(1280, header=headers.content_type)
if headers.date > datetime.utcnow() + timedelta(seconds=10):
complain(1109)
for warning in headers.warning:
if warning.code < 100 or warning.code > 299:
complain(1163, code=warning.code)
if okay(warning.date) and headers.date != warning.date:
complain(1164, code=warning.code)
for pragma in headers.pragma:
if pragma != u'no-cache':
complain(1160, pragma=pragma.item)
for protocol in headers.upgrade:
if protocol.item == u'h2':
complain(1228)
if protocol.item == upgrade.h2c and msg.is_tls:
complain(1233)
if getattr(msg, 'status', None) == st.early_hints:
# 103 (Early Hints) responses are weird in that the headers they carry
# do not apply to themselves (RFC 8297 Section 2) but only to the final
# response (and then only speculatively). For such responses, we limit
# ourselves to checks that do not rely on having a complete and
# self-consistent message header block.
return
if headers.upgrade.is_present and u'upgrade' not in headers.connection:
complain(1050)
if msg.transformed_by_proxy:
if warn.transformation_applied not in headers.warning:
complain(1191)
if headers.cache_control.no_transform:
complain(1192)
if version == http2:
for hdr in headers:
if hdr.name in [h.connection, h.transfer_encoding, h.keep_alive]:
complain(1244, header=hdr)
elif hdr.name == h.upgrade:
complain(1245)
def is_tls(self):
if okay(self.request):
return self.request.is_tls
else: # pragma: no cover
return None
def query_params(self):
# `parse_qs` returns an empty dictionary on garbage,
# so this property should be understood as "salvageable query params."
if not okay(self.effective_uri):
return {}
return parse_qs(urlparse(self.effective_uri).query)
def __iter__(self):
return iter(v for v in self.value if okay(v))
def is_okay(self):
return okay(self.value)
def check_response_itself(resp):
resp.silence(notice_id
for (notice_id, _) in resp.headers.httpolice_silence)
message.check_message(resp)
complain = resp.complain
version = resp.version
status = resp.status
headers = resp.headers
body = resp.body
# Check syntax of reason phrase.
if okay(resp.reason):
simple_parse(resp.reason,
rfc7230.reason_phrase,
complain,
1294,
place=u'reason phrase')
if not (100 <= status <= 599):
complain(1167)
if status.informational and u'close' in headers.connection:
complain(1198)
if status.informational or status == st.no_content:
if headers.transfer_encoding.is_present:
complain(1018)
if headers.content_length.is_present:
complain(1023)
for hdr in headers:
if header.is_for_response(hdr.name) == False:
complain(1064, header=hdr)
elif header.is_representation_metadata(hdr.name) and \
status.informational:
complain(1052, header=hdr)
if status == st.switching_protocols:
if headers.upgrade.is_absent:
complain(1048)
if version == http2:
complain(1246)
if status == st.no_content and body:
complain(1240)
if status == st.reset_content and body:
complain(1076)
if headers.location.is_absent:
if status == st.moved_permanently:
complain(1078)
if status == st.found:
complain(1079)
if status == st.see_other:
complain(1080)
if status == st.temporary_redirect:
complain(1084)
if status == st.permanent_redirect:
complain(1205)
if status == st.use_proxy:
complain(1082)
if status == 306:
complain(1083)
if status == st.payment_required:
complain(1088)
if status == st.method_not_allowed and headers.allow.is_absent:
complain(1089)
if status == st.request_timeout and u'close' not in headers.connection:
complain(1094)
if headers.date.is_absent and (status.successful or status.redirection
or status.client_error):
complain(1110)
if status == st.created and headers.location.is_okay and \
urlparse(headers.location.value).fragment:
complain(1111)
if headers.location.is_present and \
not status.redirection and status != st.created:
complain(1112)
if headers.retry_after.is_present and \
not status.redirection and \
status not in [st.payload_too_large, st.service_unavailable,
st.too_many_requests]:
complain(1113)
if headers.date < headers.last_modified.value:
complain(1118)
if status == st.not_modified:
for hdr in headers:
# RFC 7232 says "Last-Modified might be useful
# if the response does not have an ETag field",
# but really it doesn't hurt even if there is an ETag,
# and this is widely seen in practice.
if hdr.name in [h.etag, h.last_modified]:
continue
elif header.is_representation_metadata(hdr.name):
complain(1127, header=hdr)
if headers.content_range.is_present and \
status not in [st.partial_content, st.range_not_satisfiable]:
complain(1147)
if status == st.partial_content:
if headers.content_type == media.multipart_byteranges:
_check_multipart_byteranges(resp)
if headers.content_range.is_present:
complain(1143)
elif headers.content_range.is_absent:
complain(1138)
for direct in headers.cache_control:
if cache_directive.is_for_response(direct.item) == False:
complain(1153, directive=direct.item)
if u'no-cache' in headers.pragma:
complain(1162)
if resp.from_cache:
if headers.age.is_absent:
complain(1166)
if headers.cache_control.no_cache in [True, []]:
complain(1175)
if headers.cache_control.no_store:
complain(1176)
if status_code.is_cacheable(status) == NOT_AT_ALL:
complain(1202)
elif status_code.is_cacheable(status) == NOT_BY_DEFAULT:
if headers.expires.is_absent and headers.cache_control.is_absent:
complain(1177)
if resp.heuristic_expiration:
if headers.age > (24 * 60 * 60) and \
warn.heuristic_expiration not in headers.warning:
complain(1180)
if headers.expires.is_present:
complain(1181)
elif headers.cache_control.max_age is not None:
complain(1182)
if resp.stale:
if warn.response_is_stale not in headers.warning:
complain(1186)
if headers.cache_control.must_revalidate:
complain(1187)
for direct1, direct2 in [(cache.public, cache.no_store),
(cache.private, cache.public),
(cache.private, cache.no_store),
(cache.must_revalidate,
cache.stale_while_revalidate),
(cache.must_revalidate, cache.stale_if_error)]:
if headers.cache_control[direct1] and headers.cache_control[direct2]:
complain(1193, directive1=direct1, directive2=direct2)
for direct1, direct2 in [(cache.max_age, cache.no_cache),
(cache.max_age, cache.no_store),
(cache.s_maxage, cache.private),
(cache.s_maxage, cache.no_cache),
(cache.s_maxage, cache.no_store)]:
if headers.cache_control[direct1] and \
headers.cache_control[direct2] in [True, []]:
complain(1238, directive1=direct1, directive2=direct2)
if headers.vary != u'*' and h.host in headers.vary:
complain(1235)
if status == st.unauthorized and headers.www_authenticate.is_absent:
complain(1194)
if status == st.proxy_authentication_required and \
headers.proxy_authenticate.is_absent:
complain(1195)
for hdr in [headers.www_authenticate, headers.proxy_authenticate]:
for challenge in hdr:
if challenge.item == auth.basic:
_check_basic_challenge(resp, hdr, challenge)
if challenge.item == auth.bearer:
_check_bearer_challenge(resp, hdr, challenge)
if headers.allow.is_present and headers.accept_patch.is_present and \
m.PATCH not in headers.allow:
complain(1217)
if headers.strict_transport_security.is_okay:
if hsts.max_age not in headers.strict_transport_security:
complain(1218)
if headers.strict_transport_security.max_age == 0 and \
headers.strict_transport_security.includesubdomains:
complain(1219)
for dupe in duplicates(d.item
for d in headers.strict_transport_security):
complain(1220, directive=dupe)
for patch_type in headers.accept_patch:
if media_type.is_patch(patch_type.item) == False:
complain(1227, patch_type=patch_type.item)
if resp.transformed_by_proxy and headers.via.is_absent:
complain(1046)
if status == st.unavailable_for_legal_reasons:
if not any(rel.blocked_by in link.param.get(u'rel', [])
for link in headers.link):
complain(1243)
if headers.content_disposition.is_okay:
params = headers.content_disposition.param
for name in params.duplicates():
complain(1247, param=name)
filename = params.get(u'filename')
if filename is not None:
if contains_percent_encodes(filename):
complain(1248)
if u'"' in filename or u'\\' in filename:
# These must have been backslash-escaped.
complain(1249)
if not is_ascii(filename):
complain(1250)
filename_ext = params.get(u'filename*')
if filename_ext is not None:
if filename is None:
complain(1251)
elif params.index(u'filename*') < params.index(u'filename'):
complain(1252)
if filename_ext.charset != u'UTF-8':
complain(1255)
if headers.alt_svc.is_present:
if version == http2:
complain(1258)
if status == st.misdirected_request:
complain(1260)
def check_request(req):
"""Apply all checks to the request `req`."""
complain = req.complain
method = req.method
version = req.version
headers = req.headers
body = req.body
req.silence(notice_id
for (notice_id, in_resp) in headers.httpolice_silence
if not in_resp)
message.check_message(req)
# Check the syntax of request method and target.
parse(method, rfc7230.method, complain, 1292, place=u'request method')
_ = req.target_form
if method != method.upper() and method.upper() in known.method:
complain(1295, uppercase=Method(method.upper()))
if body and headers.content_type.is_absent:
complain(1041)
if (version in [http10, http11] and known.method.defines_body(method) and
headers.content_length.is_absent and
headers.transfer_encoding.is_absent):
complain(1021)
if (known.method.defines_body(method) == False) and (body == b'') and \
headers.content_length.is_present:
complain(1022)
if tc.chunked in headers.te:
complain(1028)
if version == http2:
if headers.te and headers.te != [u'trailers']:
complain(1244, header=headers.te)
else:
if headers.te and u'TE' not in headers.connection:
complain(1029)
if version == http11 and headers.host.is_absent:
complain(1031)
if headers.host.is_present and req.header_entries[0].name != h.host:
complain(1032)
for hdr in headers:
if known.header.is_for_request(hdr.name) == False:
complain(1063, header=hdr)
elif known.header.is_representation_metadata(hdr.name) and \
req.has_body == False:
complain(1053, header=hdr)
if body:
if method == m.GET:
complain(1056)
elif method == m.HEAD:
complain(1057)
elif method == m.DELETE:
complain(1059)
elif method == m.CONNECT:
complain(1061)
if method == m.OPTIONS and body and headers.content_type.is_absent:
complain(1062)
if headers.expect == u'100-continue' and req.has_body == False:
complain(1066)
if headers.max_forwards.is_present and method not in [m.OPTIONS, m.TRACE]:
complain(1067)
if headers.referer.is_okay:
if req.is_tls == False:
parsed = urlparse(headers.referer.value)
if parsed.scheme == u'https':
complain(1068)
if headers.user_agent.is_absent:
complain(1070)
elif headers.user_agent.is_okay:
products = [p for p in headers.user_agent if isinstance(p, Versioned)]
if products and all(known.product.is_library(p.item)
for p in products):
complain(1093, library=products[0])
for x in headers.accept_encoding:
if x.item in [cc.x_gzip, cc.x_compress] and x.param is not None:
complain(1116, coding=x.item)
if headers.if_match != u'*' and any(tag.weak for tag in headers.if_match):
complain(1120)
if method == m.HEAD:
for hdr in headers:
if known.header.is_precondition(hdr.name):
complain(1131, header=hdr)
if method in [m.CONNECT, m.OPTIONS, m.TRACE]:
for hdr in headers:
if hdr.name in [h.if_modified_since, h.if_unmodified_since,
h.if_match, h.if_none_match, h.if_range]:
complain(1130, header=hdr)
elif method not in [m.GET, m.HEAD]:
if headers.if_modified_since.is_present:
complain(1122)
if headers.range.is_present and method != m.GET:
complain(1132)
if headers.if_range.is_present and headers.range.is_absent:
complain(1134)
if isinstance(headers.if_range.value, EntityTag) and headers.if_range.weak:
complain(1135)
for direct in headers.cache_control:
if known.cache_directive.is_for_request(direct.item) == False:
complain(1152, directive=direct.item)
if direct == cache.no_cache and direct.param is not None:
complain(1159, directive=direct.item)
if headers.cache_control.no_cache and u'no-cache' not in headers.pragma:
complain(1161)
for warning in headers.warning:
if 100 <= warning.code <= 199:
complain(1165, code=warning.code)
if known.method.is_cacheable(method) == False:
for direct in headers.cache_control:
if direct.item in [cache.max_age, cache.max_stale, cache.min_fresh,
cache.no_cache, cache.no_store,
cache.only_if_cached]:
complain(1171, directive=direct)
for direct1, direct2 in [(cache.max_stale, cache.min_fresh),
(cache.stale_if_error, cache.min_fresh),
(cache.max_stale, cache.no_cache),
(cache.max_age, cache.no_cache)]:
if headers.cache_control[direct1] and headers.cache_control[direct2]:
complain(1193, directive1=direct1, directive2=direct2)
for hdr in [headers.authorization, headers.proxy_authorization]:
if hdr.is_okay:
scheme, credentials = hdr.value
if scheme == auth.basic:
_check_basic_auth(req, hdr, credentials)
elif scheme == auth.bearer:
_check_bearer_auth(req, hdr, credentials)
elif not credentials:
complain(1274, header=hdr)
if method == m.PATCH and headers.content_type.is_okay:
if known.media_type.is_patch(headers.content_type.item) == False:
complain(1213)
for protocol in headers.upgrade:
if protocol.item == upgrade.h2c:
if req.is_tls:
complain(1233)
if headers.http2_settings.is_absent:
complain(1231)
if headers.http2_settings and u'HTTP2-Settings' not in headers.connection:
complain(1230)
if headers.http2_settings.is_okay:
if not _is_urlsafe_base64(headers.http2_settings.value):
complain(1234)
if u'access_token' in req.query_params:
complain(1270)
if req.is_tls == False:
complain(1271, where=req.target)
if not headers.cache_control.no_store:
complain(1272)
if okay(req.url_encoded_data) and u'access_token' in req.url_encoded_data:
if req.is_tls == False:
complain(1271, where=req.displayable_body)
for hdr in [headers.accept, headers.accept_charset,
headers.accept_encoding, headers.accept_language]:
for (wildcard, value) in _accept_subsumptions(hdr):
complain(1276, header=hdr, wildcard=wildcard, value=value)
# No need to report more than one subsumption per header.
break
for dup_pref in duplicates(name for ((name, _), _) in headers.prefer):
complain(1285, name=dup_pref)
if headers.prefer.respond_async and known.method.is_safe(method):
complain(1287)
if headers.prefer.return_ == u'minimal' and method == m.GET:
complain(1288)
if (prefer.return_, u'minimal') in headers.prefer.without_params and \
(prefer.return_, u'representation') in headers.prefer.without_params:
complain(1289)
if (prefer.handling, u'strict') in headers.prefer.without_params and \
(prefer.handling, u'lenient') in headers.prefer.without_params:
complain(1290)
def okay(self):
return [v for v in self if okay(v)]
def check_request(req):
"""Apply all checks to the request `req`."""
req.silence(notice_id
for (notice_id, in_resp) in req.headers.httpolice_silence.okay
if not in_resp)
message.check_message(req)
_ = req.target_form # Force check.
if req.body and req.headers.content_type.is_absent:
req.complain(1041)
if (method.defines_body(req.method) and
req.headers.content_length.is_absent and
req.headers.transfer_encoding.is_absent):
req.complain(1021)
if (method.defines_body(req.method) == False) and (not req.body) and \
req.headers.content_length.is_present:
req.complain(1022)
if tc.chunked in req.headers.te:
req.complain(1028)
if req.version == http2:
if req.headers.te and req.headers.te.value != [u'trailers']:
req.complain(1244, header=req.headers.te)
else:
if req.headers.te and u'TE' not in req.headers.connection:
req.complain(1029)
if req.version == http11 and req.headers.host.is_absent:
req.complain(1031)
if req.headers.host.is_present and req.header_entries[0].name != h.host:
req.complain(1032)
for hdr in req.headers:
if header.is_for_request(hdr.name) == False:
req.complain(1063, header=hdr)
elif header.is_representation_metadata(hdr.name) and req.body == b'':
req.complain(1053, header=hdr)
if req.body:
if req.method == m.GET:
req.complain(1056)
elif req.method == m.HEAD:
req.complain(1057)
elif req.method == m.DELETE:
req.complain(1059)
elif req.method == m.CONNECT:
req.complain(1061)
if req.method == m.OPTIONS and req.body and \
req.headers.content_type.is_absent:
req.complain(1062)
if req.headers.expect == u'100-continue' and req.body == b'':
req.complain(1066)
if req.headers.max_forwards.is_present and \
req.method not in [m.OPTIONS, m.TRACE]:
req.complain(1067)
if req.headers.referer.is_okay:
if req.is_tls == False:
parsed = urlparse(req.headers.referer.value)
if parsed.scheme == u'https':
req.complain(1068)
if req.headers.user_agent.is_absent:
req.complain(1070)
elif req.headers.user_agent.is_okay:
products = [p for p in req.headers.user_agent.value
if isinstance(p, Versioned)]
if products and all(product.is_library(p.item) for p in products):
req.complain(1093, library=products[0])
for x in req.headers.accept_encoding.okay:
if x.item in [cc.x_gzip, cc.x_compress] and x.param is not None:
req.complain(1116, coding=x.item)
if req.headers.if_match.is_okay and req.headers.if_match != u'*':
if any(tag.weak for tag in req.headers.if_match.value):
req.complain(1120)
if req.method == m.HEAD:
for hdr in req.headers:
if header.is_precondition(hdr.name):
req.complain(1131, header=hdr)
if req.method in [m.CONNECT, m.OPTIONS, m.TRACE]:
for hdr in req.headers:
if hdr.name in [h.if_modified_since, h.if_unmodified_since,
h.if_match, h.if_none_match, h.if_range]:
req.complain(1130, header=hdr)
elif req.method not in [m.GET, m.HEAD]:
if req.headers.if_modified_since.is_present:
req.complain(1122)
if req.headers.range.is_present and req.method != m.GET:
req.complain(1132)
if req.headers.if_range.is_present and req.headers.range.is_absent:
req.complain(1134)
if isinstance(req.headers.if_range.value, EntityTag) and \
req.headers.if_range.value.weak:
req.complain(1135)
for d in req.headers.cache_control.okay:
if cache_directive.is_for_request(d.item) == False:
req.complain(1152, directive=d.item)
if d == cache.no_cache and d.param is not None:
req.complain(1159, directive=d.item)
if req.headers.cache_control.no_cache and \
u'no-cache' not in req.headers.pragma:
req.complain(1161)
for warning in req.headers.warning.okay:
if 100 <= warning.code < 200:
req.complain(1165, code=warning.code)
if method.is_cacheable(req.method) == False:
for direct in req.headers.cache_control.okay:
if direct.item in [cache.max_age, cache.max_stale, cache.min_fresh,
cache.no_cache, cache.no_store,
cache.only_if_cached]:
req.complain(1171, directive=direct)
for direct1, direct2 in [(cache.max_stale, cache.min_fresh),
(cache.stale_if_error, cache.min_fresh),
(cache.max_stale, cache.no_cache),
(cache.max_age, cache.no_cache)]:
if req.headers.cache_control[direct1] and \
req.headers.cache_control[direct2]:
req.complain(1193, directive1=direct1, directive2=direct2)
for hdr in [req.headers.authorization, req.headers.proxy_authorization]:
if hdr.is_okay:
scheme, credentials = hdr.value
if scheme == auth.basic:
_check_basic_auth(req, hdr, credentials)
elif scheme == auth.bearer:
_check_bearer_auth(req, hdr, credentials)
elif not credentials:
req.complain(1274, header=hdr)
if req.method == m.PATCH and req.headers.content_type.is_okay:
if media_type.is_patch(req.headers.content_type.value.item) == False:
req.complain(1213)
for proto in req.headers.upgrade.okay:
if proto.item == upgrade.h2c:
if req.is_tls:
req.complain(1233)
if req.headers.http2_settings.is_absent:
req.complain(1231)
if req.headers.http2_settings and \
u'HTTP2-Settings' not in req.headers.connection:
req.complain(1230)
if req.headers.http2_settings.is_okay:
for c in req.headers.http2_settings.value:
if c not in string.ascii_letters + string.digits + '-_':
req.complain(1234, char=c)
if u'access_token' in req.query_params:
req.complain(1270)
if req.is_tls == False:
req.complain(1271, where=req.target)
if not req.headers.cache_control.no_store:
req.complain(1272)
if okay(req.url_encoded_data) and u'access_token' in req.url_encoded_data:
if req.is_tls == False:
req.complain(1271, where=req.body)
def __iter__(self):
# pylint: disable=not-an-iterable
return iter(v for v in self.value if okay(v))
def _to_datetime(dow, d, t):
if not okay(d) or not okay(t):
return (dow, Unavailable(u'%s %s' % (d, t)))
else:
return (dow,
datetime(d.year, d.month, d.day, t.hour, t.minute, t.second))
def is_tls(self):
if not okay(self.request): # pragma: no cover
return None
return self.request.is_tls
def is_okay(self):
return okay(self.value)
def check_message(msg):
"""Run all checks that apply to any message (both request and response)."""
complain = msg.complain
version = msg.version
headers = msg.headers
for hdr in headers:
# Force parsing every header present in the message
# according to its syntax rules.
_ = hdr.value
if header.deprecated(hdr.name):
complain(1197, header=hdr)
if hdr.name.startswith(u'X-') and hdr.name not in h: # not in known
complain(1277, header=hdr)
# Force checking the payload according to various rules.
_ = msg.decoded_body
_ = msg.unicode_body
_ = msg.json_data
_ = msg.xml_data
_ = msg.multipart_data
_ = msg.url_encoded_data
if version == http11 and headers.trailer.is_present and \
tc.chunked not in headers.transfer_encoding:
# HTTP/2 supports trailers but has no notion of "chunked".
complain(1054)
for entry in msg.trailer_entries:
if entry.name not in headers.trailer:
complain(1030, header=entry)
if headers.transfer_encoding.is_present and \
headers.content_length.is_present:
complain(1020)
for opt in headers.connection:
if header.is_bad_for_connection(FieldName(opt)):
complain(1034, header=headers[FieldName(opt)])
if headers.content_type.is_okay:
if media_type.deprecated(headers.content_type.item):
complain(1035)
for dupe in headers.content_type.param.duplicates():
complain(1042, param=dupe)
if headers.content_type == media.application_json and \
u'charset' in headers.content_type.param:
complain(1280, header=headers.content_type)
if headers.upgrade.is_present and u'upgrade' not in headers.connection:
complain(1050)
if headers.date > datetime.utcnow() + timedelta(seconds=10):
complain(1109)
for warning in headers.warning:
if warning.code < 100 or warning.code > 299:
complain(1163, code=warning.code)
if okay(warning.date) and headers.date != warning.date:
complain(1164, code=warning.code)
if msg.transformed_by_proxy:
if warn.transformation_applied not in headers.warning:
complain(1191)
if headers.cache_control.no_transform:
complain(1192)
for pragma in headers.pragma:
if pragma != u'no-cache':
complain(1160, pragma=pragma.item)
if version == http2:
for hdr in headers:
if hdr.name in [h.connection, h.transfer_encoding, h.keep_alive]:
complain(1244, header=hdr)
elif hdr.name == h.upgrade:
complain(1245)
for protocol in headers.upgrade:
if protocol.item == u'h2':
complain(1228)
if protocol.item == upgrade.h2c and msg.is_tls:
complain(1233)
def _compare(self, other, op):
if isinstance(other, HeaderView):
return self.is_okay and other.is_okay and \
op(self.value, other.value)
else:
return self.is_okay and okay(other) and op(self.value, other)
def _check_bearer_challenge(resp, hdr, challenge):
# The ``Bearer`` authentication scheme is actually defined
# for proxies as well as for servers (RFC 6750 Section 1).
# Squid even seems to support it:
# http://wiki.squid-cache.org/Features/BearerAuthentication .
# However, generalizing these checks to proxies is kind of a pain,
# so for now we only handle the ``WWW-Authenticate`` series.
# If this is ever extended to proxies, the notices must be adjusted.
# Also note that some text in RFC 6750 only applies to servers
# (where it says "resource server").
if hdr.name != h.www_authenticate: # pragma: no cover
return
req = resp.request
request_has_token = None
if req:
if req.is_tls == False:
resp.complain(1263)
# Did the request contain a bearer token in one of the defined forms?
request_has_token = ((req.headers.authorization.is_okay and
req.headers.authorization.item == auth.bearer)
or (okay(req.url_encoded_data)
and u'access_token' in req.url_encoded_data)
or u'access_token' in req.query_params)
params = challenge.param
if isinstance(params, six.text_type) or not params:
# ``token68`` form or no parameters at all.
resp.complain(1264)
return
for dupe in params.duplicates():
if dupe in [
u'realm', u'scope', u'error', u'error_description',
u'error_uri'
]:
resp.complain(1265, param=dupe)
for param in [u'scope', u'error', u'error_description', u'error_uri']:
if param in params:
parser = getattr(rfc6749, param)
simple_parse(params[param],
parser,
resp.complain,
1266,
param=param,
value=params[param])
if resp.status == st.unauthorized and u'error' not in params and \
req and req.headers.authorization.is_okay and \
req.headers.authorization.item == auth.bearer:
# We don't report this if the token was passed in the URI or body,
# because the server may not implement those forms at all.
resp.complain(1267)
if u'error' in params:
error_code = params[u'error']
expected_status = {
u'invalid_request': st.bad_request,
u'invalid_token': st.unauthorized,
u'insufficient_scope': st.forbidden,
}.get(error_code)
if expected_status and resp.status != expected_status:
resp.complain(1268,
error_code=error_code,
expected_status=expected_status)
if req and req.headers.authorization.is_absent and not request_has_token:
for param in [u'error', u'error_description', u'error_uri']:
if param in params:
resp.complain(1269, param=param)
