def _pre_parse(self):
entries = []
values = []
items = self.message.headers.enumerate(self.name)
parser = header.parser_for(self.name)
for from_trailer, i, entry in items:
if from_trailer and header.is_bad_for_trailer(self.name):
self.message.complain(1026, entry=entry)
continue
entries.append(entry)
if parser is None:
parsed = entry.value
else:
(parsed,
annotations) = simple_parse(entry.value,
parser,
self.message.complain,
1000,
place=entry,
annotate_classes=known.classes)
if parsed is not Unavailable:
parsed = self._process_parsed(entry, parsed)
self.message.annotations[(from_trailer, i)] = annotations
values.append(parsed)
return entries, values
def target_form(self):
try:
target = self.target.encode('iso-8859-1')
except UnicodeError as e:
self.complain(1045, error=e)
return None
if self.method == m.CONNECT:
parser = mark(authority_form)
elif self.method == m.OPTIONS:
parser = (mark(origin_form) | mark(asterisk_form)
| mark(absolute_form))
else:
parser = mark(origin_form) | mark(absolute_form)
r = simple_parse(target,
parser,
self.complain,
1045,
place=u'request target')
if okay(r):
(symbol, _) = r
return symbol
else:
return None
def _process_parsed(self, entry, parsed):
if parsed == u'clear':
return parsed
# Parse every parameter's value according to its defined parser.
for alternative in parsed:
params = alternative.param.sequence
for i in range(len(params)):
(name, value) = params[i]
parser = alt_svc_param.parser_for(name)
if parser is not None:
value = simple_parse(value, parser,
self.message.complain, 1259,
place=entry, param=name, value=value)
params[i] = (name, value)
return parsed
def _process_directive(self, entry, directive_with_argument):
directive, argument = directive_with_argument
parser = self.knowledge_module.parser_for(directive)
if argument is None:
if self.knowledge_module.argument_required(directive):
self.message.complain(1156, entry=entry, directive=directive)
argument = Unavailable
else:
if self.knowledge_module.no_argument(directive):
self.message.complain(1157, entry=entry, directive=directive)
argument = None
elif parser is not None:
argument = simple_parse(argument, parser,
self.message.complain, 1158,
place=entry,
directive=directive, value=argument)
return Parametrized(directive, argument)
def target_form(self):
if self.method == m.CONNECT:
symbol = mark(authority_form)
elif self.method == m.OPTIONS:
symbol = (mark(origin_form) | mark(asterisk_form)
| mark(absolute_form))
else:
symbol = mark(origin_form) | mark(absolute_form)
r = simple_parse(self.target,
symbol,
self.complain,
1045,
place=u'request target')
if okay(r):
(symbol, _) = r
return symbol
else:
return r
def _process_directive(self, entry, directive_with_argument):
directive, argument = directive_with_argument
parser = self.knowledge_module.parser_for(directive)
if argument is None:
if self.knowledge_module.argument_required(directive):
self.message.complain(1156, entry=entry, directive=directive)
argument = Unavailable
else:
if self.knowledge_module.no_argument(directive):
self.message.complain(1157, entry=entry, directive=directive)
argument = None
elif parser is not None:
argument = simple_parse(argument,
parser,
self.message.complain,
1158,
place=entry,
directive=directive,
value=argument)
return Parametrized(directive, argument)
def target_form(self):
try:
target = self.target.encode('iso-8859-1')
except UnicodeError as e:
self.complain(1045, error=e)
return None
if self.method == m.CONNECT:
parser = mark(authority_form)
elif self.method == m.OPTIONS:
parser = (mark(origin_form) | mark(asterisk_form) |
mark(absolute_form))
else:
parser = mark(origin_form) | mark(absolute_form)
r = simple_parse(target, parser,
self.complain, 1045, place=u'request target')
if okay(r):
(symbol, _) = r
return symbol
else:
return None
def _process_parsed(self, entry, parsed):
if parsed == u'clear':
return parsed
# Parse every parameter's value according to its defined parser.
parsed = copy.deepcopy(parsed)
for alternative in parsed:
params = alternative.param.sequence
for i in range(len(params)):
(name, value) = params[i]
parser = alt_svc_param.parser_for(name)
if parser is not None:
value = simple_parse(value,
parser,
self.message.complain,
1259,
place=entry,
param=name,
value=value)
params[i] = (name, value)
return parsed
def check_request(req):
"""Apply all checks to the request `req`."""
complain = req.complain
method = req.method
version = req.version
headers = req.headers
body = req.body
req.silence(notice_id for (notice_id, in_resp) in headers.httpolice_silence
if not in_resp)
message.check_message(req)
# Check the syntax of request method and target.
simple_parse(method,
rfc7230.method,
complain,
1292,
place=u'request method')
_ = req.target_form
if method != method.upper() and method.upper() in m:
complain(1295, uppercase=Method(method.upper()))
if body and headers.content_type.is_absent:
complain(1041)
if (version in [http10, http11] and method_info.defines_body(method)
and headers.content_length.is_absent
and headers.transfer_encoding.is_absent):
complain(1021)
if (method_info.defines_body(method) == False) and (body == b'') and \
headers.content_length.is_present:
complain(1022)
if tc.chunked in headers.te:
complain(1028)
if version == http2:
if headers.te and headers.te != [u'trailers']:
complain(1244, header=headers.te)
else:
if headers.te and u'TE' not in headers.connection:
complain(1029)
if version == http11 and headers.host.is_absent:
complain(1031)
if headers.host.is_present and req.header_entries[0].name != h.host:
complain(1032)
for hdr in headers:
if header.is_for_request(hdr.name) == False:
complain(1063, header=hdr)
elif header.is_representation_metadata(hdr.name) and \
req.has_body == False:
complain(1053, header=hdr)
if body:
if method == m.GET:
complain(1056)
elif method == m.HEAD:
complain(1057)
elif method == m.DELETE:
complain(1059)
elif method == m.CONNECT:
complain(1061)
if method == m.OPTIONS and body and headers.content_type.is_absent:
complain(1062)
if headers.expect == u'100-continue' and req.has_body == False:
complain(1066)
if headers.max_forwards.is_present and method not in [m.OPTIONS, m.TRACE]:
complain(1067)
if headers.referer.is_okay:
if req.is_tls == False:
parsed = urlparse(headers.referer.value)
if parsed.scheme == u'https':
complain(1068)
if headers.user_agent.is_absent:
complain(1070)
elif headers.user_agent.is_okay:
products = [p for p in headers.user_agent if isinstance(p, Versioned)]
if products and all(product.is_library(p.item) for p in products):
complain(1093, library=products[0])
for x in headers.accept_encoding:
if x.item in [cc.x_gzip, cc.x_compress] and x.param is not None:
complain(1116, coding=x.item)
if headers.if_match.is_okay and headers.if_match != u'*':
if any(tag.weak for tag in headers.if_match):
complain(1120)
if method == m.HEAD:
for hdr in headers:
if header.is_precondition(hdr.name):
complain(1131, header=hdr)
if method in [m.CONNECT, m.OPTIONS, m.TRACE]:
for hdr in headers:
if hdr.name in [
h.if_modified_since, h.if_unmodified_since, h.if_match,
h.if_none_match, h.if_range
]:
complain(1130, header=hdr)
elif method not in [m.GET, m.HEAD]:
if headers.if_modified_since.is_present:
complain(1122)
if headers.range.is_present and method != m.GET:
complain(1132)
if headers.if_range.is_present and headers.range.is_absent:
complain(1134)
if isinstance(headers.if_range.value, EntityTag) and headers.if_range.weak:
complain(1135)
for direct in headers.cache_control:
if cache_directive.is_for_request(direct.item) == False:
complain(1152, directive=direct.item)
if direct == cache.no_cache and direct.param is not None:
complain(1159, directive=direct.item)
if headers.cache_control.no_cache and u'no-cache' not in headers.pragma:
complain(1161)
for warning in headers.warning:
if 100 <= warning.code <= 199:
complain(1165, code=warning.code)
if method_info.is_cacheable(method) == False:
for direct in headers.cache_control:
if direct.item in [
cache.max_age, cache.max_stale, cache.min_fresh,
cache.no_cache, cache.no_store, cache.only_if_cached
]:
complain(1171, directive=direct)
for direct1, direct2 in [(cache.max_stale, cache.min_fresh),
(cache.stale_if_error, cache.min_fresh),
(cache.max_stale, cache.no_cache),
(cache.max_age, cache.no_cache)]:
if headers.cache_control[direct1] and headers.cache_control[direct2]:
complain(1193, directive1=direct1, directive2=direct2)
for hdr in [headers.authorization, headers.proxy_authorization]:
if hdr.is_okay:
scheme, credentials = hdr.value
if scheme == auth.basic:
_check_basic_auth(req, hdr, credentials)
elif scheme == auth.bearer:
_check_bearer_auth(req, hdr, credentials)
elif not credentials:
complain(1274, header=hdr)
if method == m.PATCH and headers.content_type.is_okay:
if media_type.is_patch(headers.content_type.item) == False:
complain(1213)
for protocol in headers.upgrade:
if protocol.item == upgrade.h2c:
if req.is_tls:
complain(1233)
if headers.http2_settings.is_absent:
complain(1231)
if headers.http2_settings and u'HTTP2-Settings' not in headers.connection:
complain(1230)
if headers.http2_settings.is_okay:
if not _is_urlsafe_base64(headers.http2_settings.value):
complain(1234)
if u'access_token' in req.query_params:
complain(1270)
if req.is_tls == False:
complain(1271, where=req.target)
if not headers.cache_control.no_store:
complain(1272)
if okay(req.url_encoded_data) and u'access_token' in req.url_encoded_data:
if req.is_tls == False:
complain(1271, where=req.displayable_body)
for hdr in [
headers.accept, headers.accept_charset, headers.accept_encoding,
headers.accept_language
]:
for (wildcard, value) in _accept_subsumptions(hdr):
complain(1276, header=hdr, wildcard=wildcard, value=value)
# No need to report more than one subsumption per header.
break
for dup_pref in duplicates(name for ((name, _), _) in headers.prefer):
complain(1285, name=dup_pref)
if headers.prefer.respond_async and method_info.is_safe(method):
complain(1287)
if headers.prefer.return_ == u'minimal' and method == m.GET:
complain(1288)
if (pref.return_, u'minimal') in headers.prefer.without_params and \
(pref.return_, u'representation') in headers.prefer.without_params:
complain(1289)
if (pref.handling, u'strict') in headers.prefer.without_params and \
(pref.handling, u'lenient') in headers.prefer.without_params:
complain(1290)
def _check_bearer_challenge(resp, hdr, challenge):
# The ``Bearer`` authentication scheme is actually defined
# for proxies as well as for servers (RFC 6750 Section 1).
# Squid even seems to support it:
# http://wiki.squid-cache.org/Features/BearerAuthentication .
# However, generalizing these checks to proxies is kind of a pain,
# so for now we only handle the ``WWW-Authenticate`` series.
# If this is ever extended to proxies, the notices must be adjusted.
# Also note that some text in RFC 6750 only applies to servers
# (where it says "resource server").
if hdr.name != h.www_authenticate:
return
req = resp.request
request_has_token = None
if req:
if req.is_tls == False:
resp.complain(1263)
# Did the request contain a bearer token in one of the defined forms?
request_has_token = (
(
req.headers.authorization.is_okay and
req.headers.authorization.value.item == auth.bearer
) or
(
okay(req.url_encoded_data) and
u'access_token' in req.url_encoded_data
) or
u'access_token' in req.query_params
)
params = challenge.param
if isinstance(params, six.text_type) or not params:
# ``token68`` form or no parameters at all.
resp.complain(1264)
return
for dupe in params.duplicates():
if dupe in [u'realm', u'scope', u'error', u'error_description',
u'error_uri']:
resp.complain(1265, param=dupe)
for param in [u'scope', u'error', u'error_description', u'error_uri']:
if param in params:
parser = getattr(rfc6749, param)
simple_parse(params[param], parser,
resp.complain, 1266, param=param, value=params[param])
if resp.status == st.unauthorized and u'error' not in params and \
req and req.headers.authorization.is_okay and \
req.headers.authorization.value.item == auth.bearer:
# We don't report this if the token was passed in the URI or body,
# because the server may not implement those forms at all.
resp.complain(1267)
if u'error' in params:
error_code = params[u'error']
expected_status = {
u'invalid_request': st.bad_request,
u'invalid_token': st.unauthorized,
u'insufficient_scope': st.forbidden,
}.get(error_code)
if expected_status and resp.status != expected_status:
resp.complain(1268, error_code=error_code,
expected_status=expected_status)
if req and req.headers.authorization.is_absent and not request_has_token:
for param in [u'error', u'error_description', u'error_uri']:
if param in params:
resp.complain(1269, param=param)
def _check_bearer_challenge(resp, hdr, challenge):
# The ``Bearer`` authentication scheme is actually defined
# for proxies as well as for servers (RFC 6750 Section 1).
# Squid even seems to support it:
# http://wiki.squid-cache.org/Features/BearerAuthentication .
# However, generalizing these checks to proxies is kind of a pain,
# so for now we only handle the ``WWW-Authenticate`` series.
# If this is ever extended to proxies, the notices must be adjusted.
# Also note that some text in RFC 6750 only applies to servers
# (where it says "resource server").
if hdr.name != h.www_authenticate: # pragma: no cover
return
req = resp.request
request_has_token = None
if req:
if req.is_tls == False:
resp.complain(1263)
# Did the request contain a bearer token in one of the defined forms?
request_has_token = ((req.headers.authorization.is_okay and
req.headers.authorization.item == auth.bearer)
or (okay(req.url_encoded_data)
and u'access_token' in req.url_encoded_data)
or u'access_token' in req.query_params)
params = challenge.param
if isinstance(params, six.text_type) or not params:
# ``token68`` form or no parameters at all.
resp.complain(1264)
return
for dupe in params.duplicates():
if dupe in [
u'realm', u'scope', u'error', u'error_description',
u'error_uri'
]:
resp.complain(1265, param=dupe)
for param in [u'scope', u'error', u'error_description', u'error_uri']:
if param in params:
parser = getattr(rfc6749, param)
simple_parse(params[param],
parser,
resp.complain,
1266,
param=param,
value=params[param])
if resp.status == st.unauthorized and u'error' not in params and \
req and req.headers.authorization.is_okay and \
req.headers.authorization.item == auth.bearer:
# We don't report this if the token was passed in the URI or body,
# because the server may not implement those forms at all.
resp.complain(1267)
if u'error' in params:
error_code = params[u'error']
expected_status = {
u'invalid_request': st.bad_request,
u'invalid_token': st.unauthorized,
u'insufficient_scope': st.forbidden,
}.get(error_code)
if expected_status and resp.status != expected_status:
resp.complain(1268,
error_code=error_code,
expected_status=expected_status)
if req and req.headers.authorization.is_absent and not request_has_token:
for param in [u'error', u'error_description', u'error_uri']:
if param in params:
resp.complain(1269, param=param)
def check_response_itself(resp):
resp.silence(notice_id
for (notice_id, _) in resp.headers.httpolice_silence)
message.check_message(resp)
complain = resp.complain
version = resp.version
status = resp.status
headers = resp.headers
body = resp.body
# Check syntax of reason phrase.
if okay(resp.reason):
simple_parse(resp.reason,
rfc7230.reason_phrase,
complain,
1294,
place=u'reason phrase')
if not (100 <= status <= 599):
complain(1167)
if status.informational and u'close' in headers.connection:
complain(1198)
if status.informational or status == st.no_content:
if headers.transfer_encoding.is_present:
complain(1018)
if headers.content_length.is_present:
complain(1023)
for hdr in headers:
if header.is_for_response(hdr.name) == False:
complain(1064, header=hdr)
elif header.is_representation_metadata(hdr.name) and \
status.informational:
complain(1052, header=hdr)
if status == st.switching_protocols:
if headers.upgrade.is_absent:
complain(1048)
if version == http2:
complain(1246)
if status == st.no_content and body:
complain(1240)
if status == st.reset_content and body:
complain(1076)
if headers.location.is_absent:
if status == st.moved_permanently:
complain(1078)
if status == st.found:
complain(1079)
if status == st.see_other:
complain(1080)
if status == st.temporary_redirect:
complain(1084)
if status == st.permanent_redirect:
complain(1205)
if status == st.use_proxy:
complain(1082)
if status == 306:
complain(1083)
if status == st.payment_required:
complain(1088)
if status == st.method_not_allowed and headers.allow.is_absent:
complain(1089)
if status == st.request_timeout and u'close' not in headers.connection:
complain(1094)
if headers.date.is_absent and (status.successful or status.redirection
or status.client_error):
complain(1110)
if status == st.created and headers.location.is_okay and \
urlparse(headers.location.value).fragment:
complain(1111)
if headers.location.is_present and \
not status.redirection and status != st.created:
complain(1112)
if headers.retry_after.is_present and \
not status.redirection and \
status not in [st.payload_too_large, st.service_unavailable,
st.too_many_requests]:
complain(1113)
if headers.date < headers.last_modified.value:
complain(1118)
if status == st.not_modified:
for hdr in headers:
# RFC 7232 says "Last-Modified might be useful
# if the response does not have an ETag field",
# but really it doesn't hurt even if there is an ETag,
# and this is widely seen in practice.
if hdr.name in [h.etag, h.last_modified]:
continue
elif header.is_representation_metadata(hdr.name):
complain(1127, header=hdr)
if headers.content_range.is_present and \
status not in [st.partial_content, st.range_not_satisfiable]:
complain(1147)
if status == st.partial_content:
if headers.content_type == media.multipart_byteranges:
_check_multipart_byteranges(resp)
if headers.content_range.is_present:
complain(1143)
elif headers.content_range.is_absent:
complain(1138)
for direct in headers.cache_control:
if cache_directive.is_for_response(direct.item) == False:
complain(1153, directive=direct.item)
if u'no-cache' in headers.pragma:
complain(1162)
if resp.from_cache:
if headers.age.is_absent:
complain(1166)
if headers.cache_control.no_cache in [True, []]:
complain(1175)
if headers.cache_control.no_store:
complain(1176)
if status_code.is_cacheable(status) == NOT_AT_ALL:
complain(1202)
elif status_code.is_cacheable(status) == NOT_BY_DEFAULT:
if headers.expires.is_absent and headers.cache_control.is_absent:
complain(1177)
if resp.heuristic_expiration:
if headers.age > (24 * 60 * 60) and \
warn.heuristic_expiration not in headers.warning:
complain(1180)
if headers.expires.is_present:
complain(1181)
elif headers.cache_control.max_age is not None:
complain(1182)
if resp.stale:
if warn.response_is_stale not in headers.warning:
complain(1186)
if headers.cache_control.must_revalidate:
complain(1187)
for direct1, direct2 in [(cache.public, cache.no_store),
(cache.private, cache.public),
(cache.private, cache.no_store),
(cache.must_revalidate,
cache.stale_while_revalidate),
(cache.must_revalidate, cache.stale_if_error)]:
if headers.cache_control[direct1] and headers.cache_control[direct2]:
complain(1193, directive1=direct1, directive2=direct2)
for direct1, direct2 in [(cache.max_age, cache.no_cache),
(cache.max_age, cache.no_store),
(cache.s_maxage, cache.private),
(cache.s_maxage, cache.no_cache),
(cache.s_maxage, cache.no_store)]:
if headers.cache_control[direct1] and \
headers.cache_control[direct2] in [True, []]:
complain(1238, directive1=direct1, directive2=direct2)
if headers.vary != u'*' and h.host in headers.vary:
complain(1235)
if status == st.unauthorized and headers.www_authenticate.is_absent:
complain(1194)
if status == st.proxy_authentication_required and \
headers.proxy_authenticate.is_absent:
complain(1195)
for hdr in [headers.www_authenticate, headers.proxy_authenticate]:
for challenge in hdr:
if challenge.item == auth.basic:
_check_basic_challenge(resp, hdr, challenge)
if challenge.item == auth.bearer:
_check_bearer_challenge(resp, hdr, challenge)
if headers.allow.is_present and headers.accept_patch.is_present and \
m.PATCH not in headers.allow:
complain(1217)
if headers.strict_transport_security.is_okay:
if hsts.max_age not in headers.strict_transport_security:
complain(1218)
if headers.strict_transport_security.max_age == 0 and \
headers.strict_transport_security.includesubdomains:
complain(1219)
for dupe in duplicates(d.item
for d in headers.strict_transport_security):
complain(1220, directive=dupe)
for patch_type in headers.accept_patch:
if media_type.is_patch(patch_type.item) == False:
complain(1227, patch_type=patch_type.item)
if resp.transformed_by_proxy and headers.via.is_absent:
complain(1046)
if status == st.unavailable_for_legal_reasons:
if not any(rel.blocked_by in link.param.get(u'rel', [])
for link in headers.link):
complain(1243)
if headers.content_disposition.is_okay:
params = headers.content_disposition.param
for name in params.duplicates():
complain(1247, param=name)
filename = params.get(u'filename')
if filename is not None:
if contains_percent_encodes(filename):
complain(1248)
if u'"' in filename or u'\\' in filename:
# These must have been backslash-escaped.
complain(1249)
if not is_ascii(filename):
complain(1250)
filename_ext = params.get(u'filename*')
if filename_ext is not None:
if filename is None:
complain(1251)
elif params.index(u'filename*') < params.index(u'filename'):
complain(1252)
if filename_ext.charset != u'UTF-8':
complain(1255)
if headers.alt_svc.is_present:
if version == http2:
complain(1258)
if status == st.misdirected_request:
complain(1260)
def check_message(msg):
"""Run all checks that apply to any message (both request and response)."""
complain = msg.complain
version = msg.version
headers = msg.headers
for hdr in headers:
# Check the header name syntax.
simple_parse(hdr.name, rfc7230.field_name,
complain, 1293, header=hdr, place=u'field name')
# Force parsing every header present in the message
# according to its syntax rules.
_ = hdr.value
if header.deprecated(hdr.name):
complain(1197, header=hdr)
if hdr.name.startswith(u'X-') and hdr.name not in h: # not in known
complain(1277, header=hdr)
# Force checking the payload according to various rules.
_ = msg.decoded_body
_ = msg.unicode_body
_ = msg.json_data
_ = msg.xml_data
_ = msg.multipart_data
_ = msg.url_encoded_data
if version == http11 and headers.trailer.is_present and \
tc.chunked not in headers.transfer_encoding:
# HTTP/2 supports trailers but has no notion of "chunked".
complain(1054)
for entry in msg.trailer_entries:
if entry.name not in headers.trailer:
complain(1030, header=entry)
if headers.transfer_encoding.is_present and \
headers.content_length.is_present:
complain(1020)
for opt in headers.connection:
if header.is_bad_for_connection(FieldName(opt)):
complain(1034, header=headers[FieldName(opt)])
if headers.content_type.is_okay:
if media_type.deprecated(headers.content_type.item):
complain(1035)
for dupe in headers.content_type.param.duplicates():
complain(1042, param=dupe)
if headers.content_type == media.application_json and \
u'charset' in headers.content_type.param:
complain(1280, header=headers.content_type)
if headers.upgrade.is_present and u'upgrade' not in headers.connection:
complain(1050)
if headers.date > datetime.utcnow() + timedelta(seconds=10):
complain(1109)
for warning in headers.warning:
if warning.code < 100 or warning.code > 299:
complain(1163, code=warning.code)
if okay(warning.date) and headers.date != warning.date:
complain(1164, code=warning.code)
if msg.transformed_by_proxy:
if warn.transformation_applied not in headers.warning:
complain(1191)
if headers.cache_control.no_transform:
complain(1192)
for pragma in headers.pragma:
if pragma != u'no-cache':
complain(1160, pragma=pragma.item)
if version == http2:
for hdr in headers:
if hdr.name in [h.connection, h.transfer_encoding, h.keep_alive]:
complain(1244, header=hdr)
elif hdr.name == h.upgrade:
complain(1245)
for protocol in headers.upgrade:
if protocol.item == u'h2':
complain(1228)
if protocol.item == upgrade.h2c and msg.is_tls:
complain(1233)
def _check_alt_authority(complain, value):
return simple_parse(value,
maybe_str(uri_host) + ':' + port,
complain,
1257,
authority=value)
def _check_alt_authority(complain, value):
return simple_parse(value,
maybe_str(uri_host) + ':' + port,
complain, 1257, authority=value)