def _pre_parse(self): entries = [] values = [] items = self.message.headers.enumerate(self.name) parser = header.parser_for(self.name) for from_trailer, i, entry in items: if from_trailer and header.is_bad_for_trailer(self.name): self.message.complain(1026, entry=entry) continue entries.append(entry) if parser is None: parsed = entry.value else: (parsed, annotations) = simple_parse(entry.value, parser, self.message.complain, 1000, place=entry, annotate_classes=known.classes) if parsed is not Unavailable: parsed = self._process_parsed(entry, parsed) self.message.annotations[(from_trailer, i)] = annotations values.append(parsed) return entries, values
def target_form(self): try: target = self.target.encode('iso-8859-1') except UnicodeError as e: self.complain(1045, error=e) return None if self.method == m.CONNECT: parser = mark(authority_form) elif self.method == m.OPTIONS: parser = (mark(origin_form) | mark(asterisk_form) | mark(absolute_form)) else: parser = mark(origin_form) | mark(absolute_form) r = simple_parse(target, parser, self.complain, 1045, place=u'request target') if okay(r): (symbol, _) = r return symbol else: return None
def _process_parsed(self, entry, parsed): if parsed == u'clear': return parsed # Parse every parameter's value according to its defined parser. for alternative in parsed: params = alternative.param.sequence for i in range(len(params)): (name, value) = params[i] parser = alt_svc_param.parser_for(name) if parser is not None: value = simple_parse(value, parser, self.message.complain, 1259, place=entry, param=name, value=value) params[i] = (name, value) return parsed
def _process_directive(self, entry, directive_with_argument): directive, argument = directive_with_argument parser = self.knowledge_module.parser_for(directive) if argument is None: if self.knowledge_module.argument_required(directive): self.message.complain(1156, entry=entry, directive=directive) argument = Unavailable else: if self.knowledge_module.no_argument(directive): self.message.complain(1157, entry=entry, directive=directive) argument = None elif parser is not None: argument = simple_parse(argument, parser, self.message.complain, 1158, place=entry, directive=directive, value=argument) return Parametrized(directive, argument)
def target_form(self): if self.method == m.CONNECT: symbol = mark(authority_form) elif self.method == m.OPTIONS: symbol = (mark(origin_form) | mark(asterisk_form) | mark(absolute_form)) else: symbol = mark(origin_form) | mark(absolute_form) r = simple_parse(self.target, symbol, self.complain, 1045, place=u'request target') if okay(r): (symbol, _) = r return symbol else: return r
def _process_directive(self, entry, directive_with_argument): directive, argument = directive_with_argument parser = self.knowledge_module.parser_for(directive) if argument is None: if self.knowledge_module.argument_required(directive): self.message.complain(1156, entry=entry, directive=directive) argument = Unavailable else: if self.knowledge_module.no_argument(directive): self.message.complain(1157, entry=entry, directive=directive) argument = None elif parser is not None: argument = simple_parse(argument, parser, self.message.complain, 1158, place=entry, directive=directive, value=argument) return Parametrized(directive, argument)
def target_form(self): try: target = self.target.encode('iso-8859-1') except UnicodeError as e: self.complain(1045, error=e) return None if self.method == m.CONNECT: parser = mark(authority_form) elif self.method == m.OPTIONS: parser = (mark(origin_form) | mark(asterisk_form) | mark(absolute_form)) else: parser = mark(origin_form) | mark(absolute_form) r = simple_parse(target, parser, self.complain, 1045, place=u'request target') if okay(r): (symbol, _) = r return symbol else: return None
def _process_parsed(self, entry, parsed): if parsed == u'clear': return parsed # Parse every parameter's value according to its defined parser. parsed = copy.deepcopy(parsed) for alternative in parsed: params = alternative.param.sequence for i in range(len(params)): (name, value) = params[i] parser = alt_svc_param.parser_for(name) if parser is not None: value = simple_parse(value, parser, self.message.complain, 1259, place=entry, param=name, value=value) params[i] = (name, value) return parsed
def check_request(req): """Apply all checks to the request `req`.""" complain = req.complain method = req.method version = req.version headers = req.headers body = req.body req.silence(notice_id for (notice_id, in_resp) in headers.httpolice_silence if not in_resp) message.check_message(req) # Check the syntax of request method and target. simple_parse(method, rfc7230.method, complain, 1292, place=u'request method') _ = req.target_form if method != method.upper() and method.upper() in m: complain(1295, uppercase=Method(method.upper())) if body and headers.content_type.is_absent: complain(1041) if (version in [http10, http11] and method_info.defines_body(method) and headers.content_length.is_absent and headers.transfer_encoding.is_absent): complain(1021) if (method_info.defines_body(method) == False) and (body == b'') and \ headers.content_length.is_present: complain(1022) if tc.chunked in headers.te: complain(1028) if version == http2: if headers.te and headers.te != [u'trailers']: complain(1244, header=headers.te) else: if headers.te and u'TE' not in headers.connection: complain(1029) if version == http11 and headers.host.is_absent: complain(1031) if headers.host.is_present and req.header_entries[0].name != h.host: complain(1032) for hdr in headers: if header.is_for_request(hdr.name) == False: complain(1063, header=hdr) elif header.is_representation_metadata(hdr.name) and \ req.has_body == False: complain(1053, header=hdr) if body: if method == m.GET: complain(1056) elif method == m.HEAD: complain(1057) elif method == m.DELETE: complain(1059) elif method == m.CONNECT: complain(1061) if method == m.OPTIONS and body and headers.content_type.is_absent: complain(1062) if headers.expect == u'100-continue' and req.has_body == False: complain(1066) if headers.max_forwards.is_present and method not in [m.OPTIONS, m.TRACE]: complain(1067) if headers.referer.is_okay: if req.is_tls == False: parsed = urlparse(headers.referer.value) if parsed.scheme == u'https': complain(1068) if headers.user_agent.is_absent: complain(1070) elif headers.user_agent.is_okay: products = [p for p in headers.user_agent if isinstance(p, Versioned)] if products and all(product.is_library(p.item) for p in products): complain(1093, library=products[0]) for x in headers.accept_encoding: if x.item in [cc.x_gzip, cc.x_compress] and x.param is not None: complain(1116, coding=x.item) if headers.if_match.is_okay and headers.if_match != u'*': if any(tag.weak for tag in headers.if_match): complain(1120) if method == m.HEAD: for hdr in headers: if header.is_precondition(hdr.name): complain(1131, header=hdr) if method in [m.CONNECT, m.OPTIONS, m.TRACE]: for hdr in headers: if hdr.name in [ h.if_modified_since, h.if_unmodified_since, h.if_match, h.if_none_match, h.if_range ]: complain(1130, header=hdr) elif method not in [m.GET, m.HEAD]: if headers.if_modified_since.is_present: complain(1122) if headers.range.is_present and method != m.GET: complain(1132) if headers.if_range.is_present and headers.range.is_absent: complain(1134) if isinstance(headers.if_range.value, EntityTag) and headers.if_range.weak: complain(1135) for direct in headers.cache_control: if cache_directive.is_for_request(direct.item) == False: complain(1152, directive=direct.item) if direct == cache.no_cache and direct.param is not None: complain(1159, directive=direct.item) if headers.cache_control.no_cache and u'no-cache' not in headers.pragma: complain(1161) for warning in headers.warning: if 100 <= warning.code <= 199: complain(1165, code=warning.code) if method_info.is_cacheable(method) == False: for direct in headers.cache_control: if direct.item in [ cache.max_age, cache.max_stale, cache.min_fresh, cache.no_cache, cache.no_store, cache.only_if_cached ]: complain(1171, directive=direct) for direct1, direct2 in [(cache.max_stale, cache.min_fresh), (cache.stale_if_error, cache.min_fresh), (cache.max_stale, cache.no_cache), (cache.max_age, cache.no_cache)]: if headers.cache_control[direct1] and headers.cache_control[direct2]: complain(1193, directive1=direct1, directive2=direct2) for hdr in [headers.authorization, headers.proxy_authorization]: if hdr.is_okay: scheme, credentials = hdr.value if scheme == auth.basic: _check_basic_auth(req, hdr, credentials) elif scheme == auth.bearer: _check_bearer_auth(req, hdr, credentials) elif not credentials: complain(1274, header=hdr) if method == m.PATCH and headers.content_type.is_okay: if media_type.is_patch(headers.content_type.item) == False: complain(1213) for protocol in headers.upgrade: if protocol.item == upgrade.h2c: if req.is_tls: complain(1233) if headers.http2_settings.is_absent: complain(1231) if headers.http2_settings and u'HTTP2-Settings' not in headers.connection: complain(1230) if headers.http2_settings.is_okay: if not _is_urlsafe_base64(headers.http2_settings.value): complain(1234) if u'access_token' in req.query_params: complain(1270) if req.is_tls == False: complain(1271, where=req.target) if not headers.cache_control.no_store: complain(1272) if okay(req.url_encoded_data) and u'access_token' in req.url_encoded_data: if req.is_tls == False: complain(1271, where=req.displayable_body) for hdr in [ headers.accept, headers.accept_charset, headers.accept_encoding, headers.accept_language ]: for (wildcard, value) in _accept_subsumptions(hdr): complain(1276, header=hdr, wildcard=wildcard, value=value) # No need to report more than one subsumption per header. break for dup_pref in duplicates(name for ((name, _), _) in headers.prefer): complain(1285, name=dup_pref) if headers.prefer.respond_async and method_info.is_safe(method): complain(1287) if headers.prefer.return_ == u'minimal' and method == m.GET: complain(1288) if (pref.return_, u'minimal') in headers.prefer.without_params and \ (pref.return_, u'representation') in headers.prefer.without_params: complain(1289) if (pref.handling, u'strict') in headers.prefer.without_params and \ (pref.handling, u'lenient') in headers.prefer.without_params: complain(1290)
def _check_bearer_challenge(resp, hdr, challenge): # The ``Bearer`` authentication scheme is actually defined # for proxies as well as for servers (RFC 6750 Section 1). # Squid even seems to support it: # http://wiki.squid-cache.org/Features/BearerAuthentication . # However, generalizing these checks to proxies is kind of a pain, # so for now we only handle the ``WWW-Authenticate`` series. # If this is ever extended to proxies, the notices must be adjusted. # Also note that some text in RFC 6750 only applies to servers # (where it says "resource server"). if hdr.name != h.www_authenticate: return req = resp.request request_has_token = None if req: if req.is_tls == False: resp.complain(1263) # Did the request contain a bearer token in one of the defined forms? request_has_token = ( ( req.headers.authorization.is_okay and req.headers.authorization.value.item == auth.bearer ) or ( okay(req.url_encoded_data) and u'access_token' in req.url_encoded_data ) or u'access_token' in req.query_params ) params = challenge.param if isinstance(params, six.text_type) or not params: # ``token68`` form or no parameters at all. resp.complain(1264) return for dupe in params.duplicates(): if dupe in [u'realm', u'scope', u'error', u'error_description', u'error_uri']: resp.complain(1265, param=dupe) for param in [u'scope', u'error', u'error_description', u'error_uri']: if param in params: parser = getattr(rfc6749, param) simple_parse(params[param], parser, resp.complain, 1266, param=param, value=params[param]) if resp.status == st.unauthorized and u'error' not in params and \ req and req.headers.authorization.is_okay and \ req.headers.authorization.value.item == auth.bearer: # We don't report this if the token was passed in the URI or body, # because the server may not implement those forms at all. resp.complain(1267) if u'error' in params: error_code = params[u'error'] expected_status = { u'invalid_request': st.bad_request, u'invalid_token': st.unauthorized, u'insufficient_scope': st.forbidden, }.get(error_code) if expected_status and resp.status != expected_status: resp.complain(1268, error_code=error_code, expected_status=expected_status) if req and req.headers.authorization.is_absent and not request_has_token: for param in [u'error', u'error_description', u'error_uri']: if param in params: resp.complain(1269, param=param)
def _check_bearer_challenge(resp, hdr, challenge): # The ``Bearer`` authentication scheme is actually defined # for proxies as well as for servers (RFC 6750 Section 1). # Squid even seems to support it: # http://wiki.squid-cache.org/Features/BearerAuthentication . # However, generalizing these checks to proxies is kind of a pain, # so for now we only handle the ``WWW-Authenticate`` series. # If this is ever extended to proxies, the notices must be adjusted. # Also note that some text in RFC 6750 only applies to servers # (where it says "resource server"). if hdr.name != h.www_authenticate: # pragma: no cover return req = resp.request request_has_token = None if req: if req.is_tls == False: resp.complain(1263) # Did the request contain a bearer token in one of the defined forms? request_has_token = ((req.headers.authorization.is_okay and req.headers.authorization.item == auth.bearer) or (okay(req.url_encoded_data) and u'access_token' in req.url_encoded_data) or u'access_token' in req.query_params) params = challenge.param if isinstance(params, six.text_type) or not params: # ``token68`` form or no parameters at all. resp.complain(1264) return for dupe in params.duplicates(): if dupe in [ u'realm', u'scope', u'error', u'error_description', u'error_uri' ]: resp.complain(1265, param=dupe) for param in [u'scope', u'error', u'error_description', u'error_uri']: if param in params: parser = getattr(rfc6749, param) simple_parse(params[param], parser, resp.complain, 1266, param=param, value=params[param]) if resp.status == st.unauthorized and u'error' not in params and \ req and req.headers.authorization.is_okay and \ req.headers.authorization.item == auth.bearer: # We don't report this if the token was passed in the URI or body, # because the server may not implement those forms at all. resp.complain(1267) if u'error' in params: error_code = params[u'error'] expected_status = { u'invalid_request': st.bad_request, u'invalid_token': st.unauthorized, u'insufficient_scope': st.forbidden, }.get(error_code) if expected_status and resp.status != expected_status: resp.complain(1268, error_code=error_code, expected_status=expected_status) if req and req.headers.authorization.is_absent and not request_has_token: for param in [u'error', u'error_description', u'error_uri']: if param in params: resp.complain(1269, param=param)
def check_response_itself(resp): resp.silence(notice_id for (notice_id, _) in resp.headers.httpolice_silence) message.check_message(resp) complain = resp.complain version = resp.version status = resp.status headers = resp.headers body = resp.body # Check syntax of reason phrase. if okay(resp.reason): simple_parse(resp.reason, rfc7230.reason_phrase, complain, 1294, place=u'reason phrase') if not (100 <= status <= 599): complain(1167) if status.informational and u'close' in headers.connection: complain(1198) if status.informational or status == st.no_content: if headers.transfer_encoding.is_present: complain(1018) if headers.content_length.is_present: complain(1023) for hdr in headers: if header.is_for_response(hdr.name) == False: complain(1064, header=hdr) elif header.is_representation_metadata(hdr.name) and \ status.informational: complain(1052, header=hdr) if status == st.switching_protocols: if headers.upgrade.is_absent: complain(1048) if version == http2: complain(1246) if status == st.no_content and body: complain(1240) if status == st.reset_content and body: complain(1076) if headers.location.is_absent: if status == st.moved_permanently: complain(1078) if status == st.found: complain(1079) if status == st.see_other: complain(1080) if status == st.temporary_redirect: complain(1084) if status == st.permanent_redirect: complain(1205) if status == st.use_proxy: complain(1082) if status == 306: complain(1083) if status == st.payment_required: complain(1088) if status == st.method_not_allowed and headers.allow.is_absent: complain(1089) if status == st.request_timeout and u'close' not in headers.connection: complain(1094) if headers.date.is_absent and (status.successful or status.redirection or status.client_error): complain(1110) if status == st.created and headers.location.is_okay and \ urlparse(headers.location.value).fragment: complain(1111) if headers.location.is_present and \ not status.redirection and status != st.created: complain(1112) if headers.retry_after.is_present and \ not status.redirection and \ status not in [st.payload_too_large, st.service_unavailable, st.too_many_requests]: complain(1113) if headers.date < headers.last_modified.value: complain(1118) if status == st.not_modified: for hdr in headers: # RFC 7232 says "Last-Modified might be useful # if the response does not have an ETag field", # but really it doesn't hurt even if there is an ETag, # and this is widely seen in practice. if hdr.name in [h.etag, h.last_modified]: continue elif header.is_representation_metadata(hdr.name): complain(1127, header=hdr) if headers.content_range.is_present and \ status not in [st.partial_content, st.range_not_satisfiable]: complain(1147) if status == st.partial_content: if headers.content_type == media.multipart_byteranges: _check_multipart_byteranges(resp) if headers.content_range.is_present: complain(1143) elif headers.content_range.is_absent: complain(1138) for direct in headers.cache_control: if cache_directive.is_for_response(direct.item) == False: complain(1153, directive=direct.item) if u'no-cache' in headers.pragma: complain(1162) if resp.from_cache: if headers.age.is_absent: complain(1166) if headers.cache_control.no_cache in [True, []]: complain(1175) if headers.cache_control.no_store: complain(1176) if status_code.is_cacheable(status) == NOT_AT_ALL: complain(1202) elif status_code.is_cacheable(status) == NOT_BY_DEFAULT: if headers.expires.is_absent and headers.cache_control.is_absent: complain(1177) if resp.heuristic_expiration: if headers.age > (24 * 60 * 60) and \ warn.heuristic_expiration not in headers.warning: complain(1180) if headers.expires.is_present: complain(1181) elif headers.cache_control.max_age is not None: complain(1182) if resp.stale: if warn.response_is_stale not in headers.warning: complain(1186) if headers.cache_control.must_revalidate: complain(1187) for direct1, direct2 in [(cache.public, cache.no_store), (cache.private, cache.public), (cache.private, cache.no_store), (cache.must_revalidate, cache.stale_while_revalidate), (cache.must_revalidate, cache.stale_if_error)]: if headers.cache_control[direct1] and headers.cache_control[direct2]: complain(1193, directive1=direct1, directive2=direct2) for direct1, direct2 in [(cache.max_age, cache.no_cache), (cache.max_age, cache.no_store), (cache.s_maxage, cache.private), (cache.s_maxage, cache.no_cache), (cache.s_maxage, cache.no_store)]: if headers.cache_control[direct1] and \ headers.cache_control[direct2] in [True, []]: complain(1238, directive1=direct1, directive2=direct2) if headers.vary != u'*' and h.host in headers.vary: complain(1235) if status == st.unauthorized and headers.www_authenticate.is_absent: complain(1194) if status == st.proxy_authentication_required and \ headers.proxy_authenticate.is_absent: complain(1195) for hdr in [headers.www_authenticate, headers.proxy_authenticate]: for challenge in hdr: if challenge.item == auth.basic: _check_basic_challenge(resp, hdr, challenge) if challenge.item == auth.bearer: _check_bearer_challenge(resp, hdr, challenge) if headers.allow.is_present and headers.accept_patch.is_present and \ m.PATCH not in headers.allow: complain(1217) if headers.strict_transport_security.is_okay: if hsts.max_age not in headers.strict_transport_security: complain(1218) if headers.strict_transport_security.max_age == 0 and \ headers.strict_transport_security.includesubdomains: complain(1219) for dupe in duplicates(d.item for d in headers.strict_transport_security): complain(1220, directive=dupe) for patch_type in headers.accept_patch: if media_type.is_patch(patch_type.item) == False: complain(1227, patch_type=patch_type.item) if resp.transformed_by_proxy and headers.via.is_absent: complain(1046) if status == st.unavailable_for_legal_reasons: if not any(rel.blocked_by in link.param.get(u'rel', []) for link in headers.link): complain(1243) if headers.content_disposition.is_okay: params = headers.content_disposition.param for name in params.duplicates(): complain(1247, param=name) filename = params.get(u'filename') if filename is not None: if contains_percent_encodes(filename): complain(1248) if u'"' in filename or u'\\' in filename: # These must have been backslash-escaped. complain(1249) if not is_ascii(filename): complain(1250) filename_ext = params.get(u'filename*') if filename_ext is not None: if filename is None: complain(1251) elif params.index(u'filename*') < params.index(u'filename'): complain(1252) if filename_ext.charset != u'UTF-8': complain(1255) if headers.alt_svc.is_present: if version == http2: complain(1258) if status == st.misdirected_request: complain(1260)
def check_message(msg): """Run all checks that apply to any message (both request and response).""" complain = msg.complain version = msg.version headers = msg.headers for hdr in headers: # Check the header name syntax. simple_parse(hdr.name, rfc7230.field_name, complain, 1293, header=hdr, place=u'field name') # Force parsing every header present in the message # according to its syntax rules. _ = hdr.value if header.deprecated(hdr.name): complain(1197, header=hdr) if hdr.name.startswith(u'X-') and hdr.name not in h: # not in known complain(1277, header=hdr) # Force checking the payload according to various rules. _ = msg.decoded_body _ = msg.unicode_body _ = msg.json_data _ = msg.xml_data _ = msg.multipart_data _ = msg.url_encoded_data if version == http11 and headers.trailer.is_present and \ tc.chunked not in headers.transfer_encoding: # HTTP/2 supports trailers but has no notion of "chunked". complain(1054) for entry in msg.trailer_entries: if entry.name not in headers.trailer: complain(1030, header=entry) if headers.transfer_encoding.is_present and \ headers.content_length.is_present: complain(1020) for opt in headers.connection: if header.is_bad_for_connection(FieldName(opt)): complain(1034, header=headers[FieldName(opt)]) if headers.content_type.is_okay: if media_type.deprecated(headers.content_type.item): complain(1035) for dupe in headers.content_type.param.duplicates(): complain(1042, param=dupe) if headers.content_type == media.application_json and \ u'charset' in headers.content_type.param: complain(1280, header=headers.content_type) if headers.upgrade.is_present and u'upgrade' not in headers.connection: complain(1050) if headers.date > datetime.utcnow() + timedelta(seconds=10): complain(1109) for warning in headers.warning: if warning.code < 100 or warning.code > 299: complain(1163, code=warning.code) if okay(warning.date) and headers.date != warning.date: complain(1164, code=warning.code) if msg.transformed_by_proxy: if warn.transformation_applied not in headers.warning: complain(1191) if headers.cache_control.no_transform: complain(1192) for pragma in headers.pragma: if pragma != u'no-cache': complain(1160, pragma=pragma.item) if version == http2: for hdr in headers: if hdr.name in [h.connection, h.transfer_encoding, h.keep_alive]: complain(1244, header=hdr) elif hdr.name == h.upgrade: complain(1245) for protocol in headers.upgrade: if protocol.item == u'h2': complain(1228) if protocol.item == upgrade.h2c and msg.is_tls: complain(1233)
def _check_alt_authority(complain, value): return simple_parse(value, maybe_str(uri_host) + ':' + port, complain, 1257, authority=value)
def _check_alt_authority(complain, value): return simple_parse(value, maybe_str(uri_host) + ':' + port, complain, 1257, authority=value)