def unicode_body(self): if not okay(self.decoded_body): return self.decoded_body if not okay(self.guessed_charset): return Unavailable(self.decoded_body) # pylint: disable=no-member return self.decoded_body.decode(self.guessed_charset)
def _check_day_of_week(complain, r): (claimed_dow, r) = r if okay(r) and r.weekday() != claimed_dow: complain(1108, date=r.strftime('%Y-%m-%d'), claimed=_DAY_NAMES[claimed_dow], actual=_DAY_NAMES[r.weekday()]) return r
def _parse_response_body(resp, stream): req = resp.request # RFC 7230 section 3.3.3. if resp.status == st.switching_protocols: resp.body = b'' resp.complain(1011) stream.sane = False elif req and req.method == m.CONNECT and resp.status.successful: resp.body = b'' resp.complain(1012) stream.sane = False elif (resp.status.informational or resp.status in [st.no_content, st.not_modified] or (req and req.method == m.HEAD)): resp.body = b'' elif resp.headers.transfer_encoding: codings = resp.headers.transfer_encoding.value[:] if codings[-1] == tc.chunked: codings.pop() _parse_chunked(resp, stream) else: resp.body = stream.read() while codings and okay(resp.body): _decode_transfer_coding(resp, codings.pop()) elif resp.headers.content_length.is_present: _process_content_length(resp, stream) else: resp.body = stream.read()
def _check_multipart_byteranges(resp): if okay(resp.multipart_data): for i, part in enumerate(resp.multipart_data.get_payload()): if u'Content-Range' not in part: resp.complain(1141, part_num=(i + 1)) if u'Content-Type' not in part: resp.complain(1142, part_num=(i + 1))
def displayable_body(self): """ The payload body in a form that is appropriate for display in a message preview, along with a list of phrases explaining which transformations have been applied to arrive at that form. """ removing_te = [u'removing Transfer-Encoding'] \ if self.headers.transfer_encoding else [] removing_ce = [u'removing Content-Encoding'] \ if self.headers.content_encoding else [] decoding_charset = [u'decoding from %s' % self.guessed_charset] \ if self.guessed_charset and self.guessed_charset != 'utf-8' else [] pretty_printing = [u'pretty-printing'] if okay(self.json_data): r = json.dumps(self.json_data, indent=2, ensure_ascii=False) transforms = \ removing_te + removing_ce + decoding_charset + pretty_printing elif okay(self.unicode_body): r = self.unicode_body transforms = removing_te + removing_ce + decoding_charset elif okay(self.decoded_body): # pylint: disable=no-member r = self.decoded_body.decode('utf-8', 'replace') transforms = removing_te + removing_ce elif okay(self.body): r = self.body.decode('utf-8', 'replace') transforms = removing_te else: return self.body, [] limit = 1000 if len(r) > limit: r = r[:limit] transforms += [u'taking the first %d characters' % limit] pr = printable(r) if r != pr: r = pr transforms += [u'replacing non-printable characters ' u'with the \ufffd sign'] return r, transforms
def __init__(self, version, header_entries, body, trailer_entries=None): super(Message, self).__init__() self.version = (HTTPVersion(force_unicode(version)) if version is not None else None) self.header_entries = [HeaderEntry(k, v) for k, v in header_entries] self.body = bytes(body) if okay(body) else body self.trailer_entries = [HeaderEntry(k, v) for k, v in trailer_entries or []] self.rebuild_headers() self.annotations = {}
def json_data(self): if self.headers.content_type.is_okay and \ media_type.is_json(self.headers.content_type.value.item) and \ okay(self.unicode_body) and self.content_is_full: try: return json.loads(self.unicode_body) except ValueError as e: self.complain(1038, error=e) return Unavailable else: return None
def url_encoded_data(self): if self.headers.content_type == \ media.application_x_www_form_urlencoded and \ okay(self.decoded_body) and self.content_is_full: for char in iterbytes(self.decoded_body): if not URL_ENCODED_GOOD_CHARS[ord(char)]: self.complain(1040, char=format_chars([char])) return Unavailable(self.decoded_body) # pylint: disable=no-member return parse_qs(self.decoded_body.decode('ascii')) return None
def content_is_full(self): """Does this response carry a complete instance of its Content-Type?""" if self.status in [st.not_modified, st.early_hints]: return False if self.status == st.partial_content and \ self.headers.content_type != media.multipart_byteranges: return False if okay(self.request): return self.request.method != m.HEAD if self.body: return True return None # pragma: no cover
def decoded_body(self): """The payload body with Content-Encoding removed.""" r = self.body codings = self.headers.content_encoding.value[:] while codings and okay(r) and r: coding = codings.pop() decoder = {cc.gzip: decode_gzip, cc.x_gzip: decode_gzip, cc.deflate: decode_deflate, cc.br: decode_brotli}.get(coding) if decoder is not None: try: r = decoder(r) except Exception as e: self.complain(1037, coding=coding, error=e) r = Unavailable(r) elif okay(coding): self.complain(1036, coding=coding) r = Unavailable(r) else: r = Unavailable(r) return r
def test_tcpick_request_timeout(): [box, exch1] = load_from_tcpick('request_timeout') # The first exchange is only a box for no. 1278. assert box.request is None assert box.responses == [] assert [complaint.id for complaint in box.complaints] == [1278] # The second exchange contains only the 408 response. assert exch1.request is None [resp] = exch1.responses assert resp.status == st.request_timeout assert okay(resp.xml_data)
def url_encoded_data(self): if self.headers.content_type == \ media.application_x_www_form_urlencoded and \ okay(self.decoded_body) and self.content_is_full: for byte in six.iterbytes(self.decoded_body): if not URL_ENCODED_GOOD_BYTES[byte]: char = six.int2byte(byte) self.complain(1040, char=format_chars([char])) return Unavailable # pylint: disable=no-member return parse_qs(self.decoded_body.decode('ascii')) else: return None
def decoded_body(self): r = self.body codings = list(self.headers.content_encoding) while codings and okay(r) and r: coding = codings.pop() if coding in [cc.gzip, cc.x_gzip]: try: r = decode_gzip(r) except Exception as e: self.complain(1037, coding=coding, error=e) r = Unavailable elif coding == cc.deflate: try: r = decode_deflate(r) except Exception as e: self.complain(1037, coding=coding, error=e) r = Unavailable elif okay(coding): self.complain(1036, coding=coding) r = Unavailable else: r = Unavailable return r
def json_data(self): if self.headers.content_type.is_okay and \ known.media_type.is_json(self.headers.content_type.item) and \ okay(self.unicode_body) and self.content_is_full: try: r = json.loads(self.unicode_body) except ValueError as e: self.complain(1038, error=e) r = Unavailable(self.unicode_body) else: if self.guessed_charset not in ['ascii', 'utf-8', None]: self.complain(1281) return r return None
def _displayable_body(msg): removing_te = [u'removing Transfer-Encoding'] \ if msg.headers.transfer_encoding else [] removing_ce = [u'removing Content-Encoding'] \ if msg.headers.content_encoding else [] decoding_charset = [u'decoding from %s' % msg.guessed_charset] \ if msg.guessed_charset and msg.guessed_charset != 'utf-8' else [] pretty_printing = [u'pretty-printing'] if okay(msg.json_data): r = json.dumps(msg.json_data, indent=2, ensure_ascii=False) transforms = \ removing_te + removing_ce + decoding_charset + pretty_printing elif okay(msg.unicode_body): r = msg.unicode_body transforms = removing_te + removing_ce + decoding_charset elif okay(msg.decoded_body): r = msg.decoded_body.decode('utf-8', 'replace') transforms = removing_te + removing_ce elif okay(msg.body): r = msg.body.decode('utf-8', 'replace') transforms = removing_te else: return msg.body, [] limit = 1000 if len(r) > limit: r = r[:limit] transforms += [u'taking the first %d characters' % limit] pr = printable(r) if r != pr: r = pr transforms += [u'replacing non-printable characters ' u'with the \ufffd sign'] return r, transforms
def _process_content_length(msg, stream): n = msg.headers.content_length.value if not okay(n): msg.body = Unavailable() stream.sane = False elif n > MAX_BODY_SIZE: msg.body = Unavailable() stream.sane = False msg.complain(1298, place=msg.headers.content_length, size=n, max_size=MAX_BODY_SIZE) else: try: msg.body = stream.read(n) except ParseError as exc: msg.body = Unavailable() msg.complain(1004, error=exc)
def xml_data(self): if self.headers.content_type.is_okay and \ media_type.is_xml(self.headers.content_type.value.item) and \ okay(self.decoded_body) and self.content_is_full: try: # It's not inconceivable that a message might contain # maliciously constructed XML data, so we use `defusedxml`. return defusedxml.ElementTree.fromstring(self.decoded_body) except defusedxml.EntitiesForbidden: self.complain(1275) return Unavailable except xml.etree.ElementTree.ParseError as e: self.complain(1039, error=e) return Unavailable else: return None
def delimited_by_close(self): if self.headers.content_length.is_present or \ tc.chunked in self.headers.transfer_encoding or \ self.status.informational or \ self.status in [st.no_content, st.not_modified] or \ self.version == http2: return False if okay(self.request): if self.request.method == m.HEAD: return False if self.request.method == m.CONNECT and self.status.successful: return False if self.version in [http10, http11]: self.complain(1025) return True return None
def xml_data(self): if self.headers.content_type.is_okay and \ known.media_type.is_xml(self.headers.content_type.item) and \ okay(self.decoded_body) and self.content_is_full: try: # It's not inconceivable that a message might contain # maliciously constructed XML data, so we use `defusedxml`. return defusedxml.ElementTree.fromstring(self.decoded_body) except defusedxml.EntitiesForbidden: self.complain(1275) return Unavailable(self.decoded_body) # https://bugs.python.org/issue29896 except (xml.etree.ElementTree.ParseError, UnicodeError) as e: self.complain(1039, error=e) return Unavailable(self.decoded_body) else: return None
def guessed_charset(self): charset = 'utf-8' if self.headers.content_type.is_okay: charset = self.headers.content_type.param.get(u'charset', charset) try: codec = codecs.lookup(charset) except (LookupError, UnicodeError): return None charset = codec.name if okay(self.decoded_body): try: self.decoded_body.decode(charset) # pylint: disable=no-member except UnicodeError: return None return charset
def multipart_data(self): ctype = self.headers.content_type if ctype.is_okay and media_type.is_multipart(ctype.value.item) and \ okay(self.decoded_body) and self.content_is_full: # All multipart media types obey the same general syntax # specified in RFC 2046 Section 5.1, # and should be parseable as email message payloads. multipart_code = (b'Content-Type: ' + ctype.entries[0].value + b'\r\n\r\n' + self.decoded_body) parsed = parse_email_message(multipart_code) for d in parsed.defects: if isinstance(d, email.errors.NoBoundaryInMultipartDefect): self.complain(1139) elif isinstance(d, email.errors.StartBoundaryNotFoundDefect): self.complain(1140) return parsed if parsed.is_multipart() else Unavailable else: return None
def _parse_response_body(resp, stream): req = resp.request # RFC 7230 section 3.3.3. if resp.status == st.switching_protocols: resp.body = b'' resp.complain(1011) stream.sane = False elif req and req.method == m.CONNECT and resp.status.successful: resp.body = b'' resp.complain(1012) stream.sane = False elif (resp.status.informational or resp.status in [st.no_content, st.not_modified] or (req and req.method == m.HEAD)): resp.body = b'' elif resp.headers.transfer_encoding: codings = list(resp.headers.transfer_encoding) if codings[-1] == tc.chunked: codings.pop() _parse_chunked(resp, stream) else: resp.body = stream.consume_rest() while codings and okay(resp.body): _decode_transfer_coding(resp, codings.pop()) elif resp.headers.content_length.is_present: n = resp.headers.content_length.value if n is Unavailable: resp.body = Unavailable stream.sane = False else: try: resp.body = stream.consume_n_bytes(n) except ParseError: resp.body = Unavailable resp.complain(1004) stream.sane = False else: resp.body = stream.consume_rest()
def _parse_request_body(req, stream): # RFC 7230 section 3.3.3. if req.headers.transfer_encoding: codings = req.headers.transfer_encoding.value[:] if codings.pop() == tc.chunked: _parse_chunked(req, stream) else: req.body = Unavailable() req.complain(1001) stream.sane = False while codings and okay(req.body): _decode_transfer_coding(req, codings.pop()) elif req.headers.content_length: _process_content_length(req, stream) else: req.body = b''
def _decode_transfer_coding(msg, coding): if coding == tc.chunked: # The outermost chunked has already been peeled off at this point. msg.complain(1002) msg.body = Unavailable(msg.body) elif coding in [tc.gzip, tc.x_gzip]: try: msg.body = decode_gzip(msg.body) except Exception as e: msg.complain(1027, coding=coding, error=e) msg.body = Unavailable(msg.body) elif coding == tc.deflate: try: msg.body = decode_deflate(msg.body) except Exception as e: msg.complain(1027, coding=coding, error=e) msg.body = Unavailable(msg.body) else: if okay(coding): msg.complain(1003, coding=coding) msg.body = Unavailable(msg.body)
def target_form(self): try: target = self.target.encode('iso-8859-1') except UnicodeError as e: self.complain(1045, error=e) return None if self.method == m.CONNECT: parser = mark(authority_form) elif self.method == m.OPTIONS: parser = (mark(origin_form) | mark(asterisk_form) | mark(absolute_form)) else: parser = mark(origin_form) | mark(absolute_form) r = simple_parse(target, parser, self.complain, 1045, place=u'request target') if okay(r): (symbol, _) = r return symbol else: return None
def _compare(self, other, op): # It would be nice to be able to compare headers by values, as in:: # # resp.headers.last_modified == req.headers.if_modified_since # # Unfortunately, there are places # (such as :meth:`httpolice.blackboard.Blackboard.complain`) # where we need header-to-header equality to be less magical. # And if we can't do this magic for equality, # there's no sense in doing it for other operations. # So we just say that comparing headers to headers is `NotImplemented` # (fall back to comparing their object identities). # # Now, the following form still works:: # # resp.headers.last_modified == req.headers.if_modified_since.value # # so we don't lose all that much. if isinstance(other, HeaderView): return NotImplemented return self.is_okay and okay(other) and op(self.value, other)
def check_message(msg): """Run all checks that apply to any message (both request and response).""" complain = msg.complain version = msg.version headers = msg.headers x_prefixed = [] for hdr in headers: # Check the header name syntax. parse(hdr.name, rfc7230.field_name, complain, 1293, header=hdr, place=u'field name') # Force parsing every header present in the message # according to its syntax rules. _ = hdr.value if known.header.is_deprecated(hdr.name): complain(1197, header=hdr) if hdr.name.startswith(u'X-') and hdr.name not in known.header: x_prefixed.append(hdr) if x_prefixed: complain(1277, headers=x_prefixed) # Force checking the payload according to various rules. _ = msg.decoded_body _ = msg.unicode_body _ = msg.json_data _ = msg.xml_data _ = msg.multipart_data _ = msg.url_encoded_data if version == http11 and headers.trailer.is_present and \ tc.chunked not in headers.transfer_encoding: # HTTP/2 supports trailers but has no notion of "chunked". complain(1054) for entry in msg.trailer_entries: if entry.name not in headers.trailer: complain(1030, header=entry) if headers.transfer_encoding.is_present and \ headers.content_length.is_present: complain(1020) for opt in headers.connection: if known.header.is_bad_for_connection(FieldName(opt)): complain(1034, header=headers[FieldName(opt)]) if headers.content_type.is_okay: if known.media_type.is_deprecated(headers.content_type.item): complain(1035) for dupe in headers.content_type.param.duplicates(): complain(1042, param=dupe) if headers.content_type == media.application_json and \ u'charset' in headers.content_type.param: complain(1280, header=headers.content_type) if headers.date > datetime.utcnow() + timedelta(seconds=10): complain(1109) for warning in headers.warning: if warning.code < 100 or warning.code > 299: complain(1163, code=warning.code) if okay(warning.date) and headers.date != warning.date: complain(1164, code=warning.code) for pragma in headers.pragma: if pragma != u'no-cache': complain(1160, pragma=pragma.item) for protocol in headers.upgrade: if protocol.item == u'h2': complain(1228) if protocol.item == upgrade.h2c and msg.is_tls: complain(1233) if getattr(msg, 'status', None) == st.early_hints: # 103 (Early Hints) responses are weird in that the headers they carry # do not apply to themselves (RFC 8297 Section 2) but only to the final # response (and then only speculatively). For such responses, we limit # ourselves to checks that do not rely on having a complete and # self-consistent message header block. return if headers.upgrade.is_present and u'upgrade' not in headers.connection: complain(1050) if msg.transformed_by_proxy: if warn.transformation_applied not in headers.warning: complain(1191) if headers.cache_control.no_transform: complain(1192) if version == http2: for hdr in headers: if hdr.name in [h.connection, h.transfer_encoding, h.keep_alive]: complain(1244, header=hdr) elif hdr.name == h.upgrade: complain(1245)
def is_tls(self): if okay(self.request): return self.request.is_tls else: # pragma: no cover return None
def query_params(self): # `parse_qs` returns an empty dictionary on garbage, # so this property should be understood as "salvageable query params." if not okay(self.effective_uri): return {} return parse_qs(urlparse(self.effective_uri).query)
def __iter__(self): return iter(v for v in self.value if okay(v))
def is_okay(self): return okay(self.value)
def check_response_itself(resp): resp.silence(notice_id for (notice_id, _) in resp.headers.httpolice_silence) message.check_message(resp) complain = resp.complain version = resp.version status = resp.status headers = resp.headers body = resp.body # Check syntax of reason phrase. if okay(resp.reason): simple_parse(resp.reason, rfc7230.reason_phrase, complain, 1294, place=u'reason phrase') if not (100 <= status <= 599): complain(1167) if status.informational and u'close' in headers.connection: complain(1198) if status.informational or status == st.no_content: if headers.transfer_encoding.is_present: complain(1018) if headers.content_length.is_present: complain(1023) for hdr in headers: if header.is_for_response(hdr.name) == False: complain(1064, header=hdr) elif header.is_representation_metadata(hdr.name) and \ status.informational: complain(1052, header=hdr) if status == st.switching_protocols: if headers.upgrade.is_absent: complain(1048) if version == http2: complain(1246) if status == st.no_content and body: complain(1240) if status == st.reset_content and body: complain(1076) if headers.location.is_absent: if status == st.moved_permanently: complain(1078) if status == st.found: complain(1079) if status == st.see_other: complain(1080) if status == st.temporary_redirect: complain(1084) if status == st.permanent_redirect: complain(1205) if status == st.use_proxy: complain(1082) if status == 306: complain(1083) if status == st.payment_required: complain(1088) if status == st.method_not_allowed and headers.allow.is_absent: complain(1089) if status == st.request_timeout and u'close' not in headers.connection: complain(1094) if headers.date.is_absent and (status.successful or status.redirection or status.client_error): complain(1110) if status == st.created and headers.location.is_okay and \ urlparse(headers.location.value).fragment: complain(1111) if headers.location.is_present and \ not status.redirection and status != st.created: complain(1112) if headers.retry_after.is_present and \ not status.redirection and \ status not in [st.payload_too_large, st.service_unavailable, st.too_many_requests]: complain(1113) if headers.date < headers.last_modified.value: complain(1118) if status == st.not_modified: for hdr in headers: # RFC 7232 says "Last-Modified might be useful # if the response does not have an ETag field", # but really it doesn't hurt even if there is an ETag, # and this is widely seen in practice. if hdr.name in [h.etag, h.last_modified]: continue elif header.is_representation_metadata(hdr.name): complain(1127, header=hdr) if headers.content_range.is_present and \ status not in [st.partial_content, st.range_not_satisfiable]: complain(1147) if status == st.partial_content: if headers.content_type == media.multipart_byteranges: _check_multipart_byteranges(resp) if headers.content_range.is_present: complain(1143) elif headers.content_range.is_absent: complain(1138) for direct in headers.cache_control: if cache_directive.is_for_response(direct.item) == False: complain(1153, directive=direct.item) if u'no-cache' in headers.pragma: complain(1162) if resp.from_cache: if headers.age.is_absent: complain(1166) if headers.cache_control.no_cache in [True, []]: complain(1175) if headers.cache_control.no_store: complain(1176) if status_code.is_cacheable(status) == NOT_AT_ALL: complain(1202) elif status_code.is_cacheable(status) == NOT_BY_DEFAULT: if headers.expires.is_absent and headers.cache_control.is_absent: complain(1177) if resp.heuristic_expiration: if headers.age > (24 * 60 * 60) and \ warn.heuristic_expiration not in headers.warning: complain(1180) if headers.expires.is_present: complain(1181) elif headers.cache_control.max_age is not None: complain(1182) if resp.stale: if warn.response_is_stale not in headers.warning: complain(1186) if headers.cache_control.must_revalidate: complain(1187) for direct1, direct2 in [(cache.public, cache.no_store), (cache.private, cache.public), (cache.private, cache.no_store), (cache.must_revalidate, cache.stale_while_revalidate), (cache.must_revalidate, cache.stale_if_error)]: if headers.cache_control[direct1] and headers.cache_control[direct2]: complain(1193, directive1=direct1, directive2=direct2) for direct1, direct2 in [(cache.max_age, cache.no_cache), (cache.max_age, cache.no_store), (cache.s_maxage, cache.private), (cache.s_maxage, cache.no_cache), (cache.s_maxage, cache.no_store)]: if headers.cache_control[direct1] and \ headers.cache_control[direct2] in [True, []]: complain(1238, directive1=direct1, directive2=direct2) if headers.vary != u'*' and h.host in headers.vary: complain(1235) if status == st.unauthorized and headers.www_authenticate.is_absent: complain(1194) if status == st.proxy_authentication_required and \ headers.proxy_authenticate.is_absent: complain(1195) for hdr in [headers.www_authenticate, headers.proxy_authenticate]: for challenge in hdr: if challenge.item == auth.basic: _check_basic_challenge(resp, hdr, challenge) if challenge.item == auth.bearer: _check_bearer_challenge(resp, hdr, challenge) if headers.allow.is_present and headers.accept_patch.is_present and \ m.PATCH not in headers.allow: complain(1217) if headers.strict_transport_security.is_okay: if hsts.max_age not in headers.strict_transport_security: complain(1218) if headers.strict_transport_security.max_age == 0 and \ headers.strict_transport_security.includesubdomains: complain(1219) for dupe in duplicates(d.item for d in headers.strict_transport_security): complain(1220, directive=dupe) for patch_type in headers.accept_patch: if media_type.is_patch(patch_type.item) == False: complain(1227, patch_type=patch_type.item) if resp.transformed_by_proxy and headers.via.is_absent: complain(1046) if status == st.unavailable_for_legal_reasons: if not any(rel.blocked_by in link.param.get(u'rel', []) for link in headers.link): complain(1243) if headers.content_disposition.is_okay: params = headers.content_disposition.param for name in params.duplicates(): complain(1247, param=name) filename = params.get(u'filename') if filename is not None: if contains_percent_encodes(filename): complain(1248) if u'"' in filename or u'\\' in filename: # These must have been backslash-escaped. complain(1249) if not is_ascii(filename): complain(1250) filename_ext = params.get(u'filename*') if filename_ext is not None: if filename is None: complain(1251) elif params.index(u'filename*') < params.index(u'filename'): complain(1252) if filename_ext.charset != u'UTF-8': complain(1255) if headers.alt_svc.is_present: if version == http2: complain(1258) if status == st.misdirected_request: complain(1260)
def check_request(req): """Apply all checks to the request `req`.""" complain = req.complain method = req.method version = req.version headers = req.headers body = req.body req.silence(notice_id for (notice_id, in_resp) in headers.httpolice_silence if not in_resp) message.check_message(req) # Check the syntax of request method and target. parse(method, rfc7230.method, complain, 1292, place=u'request method') _ = req.target_form if method != method.upper() and method.upper() in known.method: complain(1295, uppercase=Method(method.upper())) if body and headers.content_type.is_absent: complain(1041) if (version in [http10, http11] and known.method.defines_body(method) and headers.content_length.is_absent and headers.transfer_encoding.is_absent): complain(1021) if (known.method.defines_body(method) == False) and (body == b'') and \ headers.content_length.is_present: complain(1022) if tc.chunked in headers.te: complain(1028) if version == http2: if headers.te and headers.te != [u'trailers']: complain(1244, header=headers.te) else: if headers.te and u'TE' not in headers.connection: complain(1029) if version == http11 and headers.host.is_absent: complain(1031) if headers.host.is_present and req.header_entries[0].name != h.host: complain(1032) for hdr in headers: if known.header.is_for_request(hdr.name) == False: complain(1063, header=hdr) elif known.header.is_representation_metadata(hdr.name) and \ req.has_body == False: complain(1053, header=hdr) if body: if method == m.GET: complain(1056) elif method == m.HEAD: complain(1057) elif method == m.DELETE: complain(1059) elif method == m.CONNECT: complain(1061) if method == m.OPTIONS and body and headers.content_type.is_absent: complain(1062) if headers.expect == u'100-continue' and req.has_body == False: complain(1066) if headers.max_forwards.is_present and method not in [m.OPTIONS, m.TRACE]: complain(1067) if headers.referer.is_okay: if req.is_tls == False: parsed = urlparse(headers.referer.value) if parsed.scheme == u'https': complain(1068) if headers.user_agent.is_absent: complain(1070) elif headers.user_agent.is_okay: products = [p for p in headers.user_agent if isinstance(p, Versioned)] if products and all(known.product.is_library(p.item) for p in products): complain(1093, library=products[0]) for x in headers.accept_encoding: if x.item in [cc.x_gzip, cc.x_compress] and x.param is not None: complain(1116, coding=x.item) if headers.if_match != u'*' and any(tag.weak for tag in headers.if_match): complain(1120) if method == m.HEAD: for hdr in headers: if known.header.is_precondition(hdr.name): complain(1131, header=hdr) if method in [m.CONNECT, m.OPTIONS, m.TRACE]: for hdr in headers: if hdr.name in [h.if_modified_since, h.if_unmodified_since, h.if_match, h.if_none_match, h.if_range]: complain(1130, header=hdr) elif method not in [m.GET, m.HEAD]: if headers.if_modified_since.is_present: complain(1122) if headers.range.is_present and method != m.GET: complain(1132) if headers.if_range.is_present and headers.range.is_absent: complain(1134) if isinstance(headers.if_range.value, EntityTag) and headers.if_range.weak: complain(1135) for direct in headers.cache_control: if known.cache_directive.is_for_request(direct.item) == False: complain(1152, directive=direct.item) if direct == cache.no_cache and direct.param is not None: complain(1159, directive=direct.item) if headers.cache_control.no_cache and u'no-cache' not in headers.pragma: complain(1161) for warning in headers.warning: if 100 <= warning.code <= 199: complain(1165, code=warning.code) if known.method.is_cacheable(method) == False: for direct in headers.cache_control: if direct.item in [cache.max_age, cache.max_stale, cache.min_fresh, cache.no_cache, cache.no_store, cache.only_if_cached]: complain(1171, directive=direct) for direct1, direct2 in [(cache.max_stale, cache.min_fresh), (cache.stale_if_error, cache.min_fresh), (cache.max_stale, cache.no_cache), (cache.max_age, cache.no_cache)]: if headers.cache_control[direct1] and headers.cache_control[direct2]: complain(1193, directive1=direct1, directive2=direct2) for hdr in [headers.authorization, headers.proxy_authorization]: if hdr.is_okay: scheme, credentials = hdr.value if scheme == auth.basic: _check_basic_auth(req, hdr, credentials) elif scheme == auth.bearer: _check_bearer_auth(req, hdr, credentials) elif not credentials: complain(1274, header=hdr) if method == m.PATCH and headers.content_type.is_okay: if known.media_type.is_patch(headers.content_type.item) == False: complain(1213) for protocol in headers.upgrade: if protocol.item == upgrade.h2c: if req.is_tls: complain(1233) if headers.http2_settings.is_absent: complain(1231) if headers.http2_settings and u'HTTP2-Settings' not in headers.connection: complain(1230) if headers.http2_settings.is_okay: if not _is_urlsafe_base64(headers.http2_settings.value): complain(1234) if u'access_token' in req.query_params: complain(1270) if req.is_tls == False: complain(1271, where=req.target) if not headers.cache_control.no_store: complain(1272) if okay(req.url_encoded_data) and u'access_token' in req.url_encoded_data: if req.is_tls == False: complain(1271, where=req.displayable_body) for hdr in [headers.accept, headers.accept_charset, headers.accept_encoding, headers.accept_language]: for (wildcard, value) in _accept_subsumptions(hdr): complain(1276, header=hdr, wildcard=wildcard, value=value) # No need to report more than one subsumption per header. break for dup_pref in duplicates(name for ((name, _), _) in headers.prefer): complain(1285, name=dup_pref) if headers.prefer.respond_async and known.method.is_safe(method): complain(1287) if headers.prefer.return_ == u'minimal' and method == m.GET: complain(1288) if (prefer.return_, u'minimal') in headers.prefer.without_params and \ (prefer.return_, u'representation') in headers.prefer.without_params: complain(1289) if (prefer.handling, u'strict') in headers.prefer.without_params and \ (prefer.handling, u'lenient') in headers.prefer.without_params: complain(1290)
def okay(self): return [v for v in self if okay(v)]
def check_request(req): """Apply all checks to the request `req`.""" req.silence(notice_id for (notice_id, in_resp) in req.headers.httpolice_silence.okay if not in_resp) message.check_message(req) _ = req.target_form # Force check. if req.body and req.headers.content_type.is_absent: req.complain(1041) if (method.defines_body(req.method) and req.headers.content_length.is_absent and req.headers.transfer_encoding.is_absent): req.complain(1021) if (method.defines_body(req.method) == False) and (not req.body) and \ req.headers.content_length.is_present: req.complain(1022) if tc.chunked in req.headers.te: req.complain(1028) if req.version == http2: if req.headers.te and req.headers.te.value != [u'trailers']: req.complain(1244, header=req.headers.te) else: if req.headers.te and u'TE' not in req.headers.connection: req.complain(1029) if req.version == http11 and req.headers.host.is_absent: req.complain(1031) if req.headers.host.is_present and req.header_entries[0].name != h.host: req.complain(1032) for hdr in req.headers: if header.is_for_request(hdr.name) == False: req.complain(1063, header=hdr) elif header.is_representation_metadata(hdr.name) and req.body == b'': req.complain(1053, header=hdr) if req.body: if req.method == m.GET: req.complain(1056) elif req.method == m.HEAD: req.complain(1057) elif req.method == m.DELETE: req.complain(1059) elif req.method == m.CONNECT: req.complain(1061) if req.method == m.OPTIONS and req.body and \ req.headers.content_type.is_absent: req.complain(1062) if req.headers.expect == u'100-continue' and req.body == b'': req.complain(1066) if req.headers.max_forwards.is_present and \ req.method not in [m.OPTIONS, m.TRACE]: req.complain(1067) if req.headers.referer.is_okay: if req.is_tls == False: parsed = urlparse(req.headers.referer.value) if parsed.scheme == u'https': req.complain(1068) if req.headers.user_agent.is_absent: req.complain(1070) elif req.headers.user_agent.is_okay: products = [p for p in req.headers.user_agent.value if isinstance(p, Versioned)] if products and all(product.is_library(p.item) for p in products): req.complain(1093, library=products[0]) for x in req.headers.accept_encoding.okay: if x.item in [cc.x_gzip, cc.x_compress] and x.param is not None: req.complain(1116, coding=x.item) if req.headers.if_match.is_okay and req.headers.if_match != u'*': if any(tag.weak for tag in req.headers.if_match.value): req.complain(1120) if req.method == m.HEAD: for hdr in req.headers: if header.is_precondition(hdr.name): req.complain(1131, header=hdr) if req.method in [m.CONNECT, m.OPTIONS, m.TRACE]: for hdr in req.headers: if hdr.name in [h.if_modified_since, h.if_unmodified_since, h.if_match, h.if_none_match, h.if_range]: req.complain(1130, header=hdr) elif req.method not in [m.GET, m.HEAD]: if req.headers.if_modified_since.is_present: req.complain(1122) if req.headers.range.is_present and req.method != m.GET: req.complain(1132) if req.headers.if_range.is_present and req.headers.range.is_absent: req.complain(1134) if isinstance(req.headers.if_range.value, EntityTag) and \ req.headers.if_range.value.weak: req.complain(1135) for d in req.headers.cache_control.okay: if cache_directive.is_for_request(d.item) == False: req.complain(1152, directive=d.item) if d == cache.no_cache and d.param is not None: req.complain(1159, directive=d.item) if req.headers.cache_control.no_cache and \ u'no-cache' not in req.headers.pragma: req.complain(1161) for warning in req.headers.warning.okay: if 100 <= warning.code < 200: req.complain(1165, code=warning.code) if method.is_cacheable(req.method) == False: for direct in req.headers.cache_control.okay: if direct.item in [cache.max_age, cache.max_stale, cache.min_fresh, cache.no_cache, cache.no_store, cache.only_if_cached]: req.complain(1171, directive=direct) for direct1, direct2 in [(cache.max_stale, cache.min_fresh), (cache.stale_if_error, cache.min_fresh), (cache.max_stale, cache.no_cache), (cache.max_age, cache.no_cache)]: if req.headers.cache_control[direct1] and \ req.headers.cache_control[direct2]: req.complain(1193, directive1=direct1, directive2=direct2) for hdr in [req.headers.authorization, req.headers.proxy_authorization]: if hdr.is_okay: scheme, credentials = hdr.value if scheme == auth.basic: _check_basic_auth(req, hdr, credentials) elif scheme == auth.bearer: _check_bearer_auth(req, hdr, credentials) elif not credentials: req.complain(1274, header=hdr) if req.method == m.PATCH and req.headers.content_type.is_okay: if media_type.is_patch(req.headers.content_type.value.item) == False: req.complain(1213) for proto in req.headers.upgrade.okay: if proto.item == upgrade.h2c: if req.is_tls: req.complain(1233) if req.headers.http2_settings.is_absent: req.complain(1231) if req.headers.http2_settings and \ u'HTTP2-Settings' not in req.headers.connection: req.complain(1230) if req.headers.http2_settings.is_okay: for c in req.headers.http2_settings.value: if c not in string.ascii_letters + string.digits + '-_': req.complain(1234, char=c) if u'access_token' in req.query_params: req.complain(1270) if req.is_tls == False: req.complain(1271, where=req.target) if not req.headers.cache_control.no_store: req.complain(1272) if okay(req.url_encoded_data) and u'access_token' in req.url_encoded_data: if req.is_tls == False: req.complain(1271, where=req.body)
def __iter__(self): # pylint: disable=not-an-iterable return iter(v for v in self.value if okay(v))
def _to_datetime(dow, d, t): if not okay(d) or not okay(t): return (dow, Unavailable(u'%s %s' % (d, t))) else: return (dow, datetime(d.year, d.month, d.day, t.hour, t.minute, t.second))
def is_tls(self): if not okay(self.request): # pragma: no cover return None return self.request.is_tls
def is_okay(self): return okay(self.value)
def check_message(msg): """Run all checks that apply to any message (both request and response).""" complain = msg.complain version = msg.version headers = msg.headers for hdr in headers: # Force parsing every header present in the message # according to its syntax rules. _ = hdr.value if header.deprecated(hdr.name): complain(1197, header=hdr) if hdr.name.startswith(u'X-') and hdr.name not in h: # not in known complain(1277, header=hdr) # Force checking the payload according to various rules. _ = msg.decoded_body _ = msg.unicode_body _ = msg.json_data _ = msg.xml_data _ = msg.multipart_data _ = msg.url_encoded_data if version == http11 and headers.trailer.is_present and \ tc.chunked not in headers.transfer_encoding: # HTTP/2 supports trailers but has no notion of "chunked". complain(1054) for entry in msg.trailer_entries: if entry.name not in headers.trailer: complain(1030, header=entry) if headers.transfer_encoding.is_present and \ headers.content_length.is_present: complain(1020) for opt in headers.connection: if header.is_bad_for_connection(FieldName(opt)): complain(1034, header=headers[FieldName(opt)]) if headers.content_type.is_okay: if media_type.deprecated(headers.content_type.item): complain(1035) for dupe in headers.content_type.param.duplicates(): complain(1042, param=dupe) if headers.content_type == media.application_json and \ u'charset' in headers.content_type.param: complain(1280, header=headers.content_type) if headers.upgrade.is_present and u'upgrade' not in headers.connection: complain(1050) if headers.date > datetime.utcnow() + timedelta(seconds=10): complain(1109) for warning in headers.warning: if warning.code < 100 or warning.code > 299: complain(1163, code=warning.code) if okay(warning.date) and headers.date != warning.date: complain(1164, code=warning.code) if msg.transformed_by_proxy: if warn.transformation_applied not in headers.warning: complain(1191) if headers.cache_control.no_transform: complain(1192) for pragma in headers.pragma: if pragma != u'no-cache': complain(1160, pragma=pragma.item) if version == http2: for hdr in headers: if hdr.name in [h.connection, h.transfer_encoding, h.keep_alive]: complain(1244, header=hdr) elif hdr.name == h.upgrade: complain(1245) for protocol in headers.upgrade: if protocol.item == u'h2': complain(1228) if protocol.item == upgrade.h2c and msg.is_tls: complain(1233)
def _compare(self, other, op): if isinstance(other, HeaderView): return self.is_okay and other.is_okay and \ op(self.value, other.value) else: return self.is_okay and okay(other) and op(self.value, other)
def _check_bearer_challenge(resp, hdr, challenge): # The ``Bearer`` authentication scheme is actually defined # for proxies as well as for servers (RFC 6750 Section 1). # Squid even seems to support it: # http://wiki.squid-cache.org/Features/BearerAuthentication . # However, generalizing these checks to proxies is kind of a pain, # so for now we only handle the ``WWW-Authenticate`` series. # If this is ever extended to proxies, the notices must be adjusted. # Also note that some text in RFC 6750 only applies to servers # (where it says "resource server"). if hdr.name != h.www_authenticate: # pragma: no cover return req = resp.request request_has_token = None if req: if req.is_tls == False: resp.complain(1263) # Did the request contain a bearer token in one of the defined forms? request_has_token = ((req.headers.authorization.is_okay and req.headers.authorization.item == auth.bearer) or (okay(req.url_encoded_data) and u'access_token' in req.url_encoded_data) or u'access_token' in req.query_params) params = challenge.param if isinstance(params, six.text_type) or not params: # ``token68`` form or no parameters at all. resp.complain(1264) return for dupe in params.duplicates(): if dupe in [ u'realm', u'scope', u'error', u'error_description', u'error_uri' ]: resp.complain(1265, param=dupe) for param in [u'scope', u'error', u'error_description', u'error_uri']: if param in params: parser = getattr(rfc6749, param) simple_parse(params[param], parser, resp.complain, 1266, param=param, value=params[param]) if resp.status == st.unauthorized and u'error' not in params and \ req and req.headers.authorization.is_okay and \ req.headers.authorization.item == auth.bearer: # We don't report this if the token was passed in the URI or body, # because the server may not implement those forms at all. resp.complain(1267) if u'error' in params: error_code = params[u'error'] expected_status = { u'invalid_request': st.bad_request, u'invalid_token': st.unauthorized, u'insufficient_scope': st.forbidden, }.get(error_code) if expected_status and resp.status != expected_status: resp.complain(1268, error_code=error_code, expected_status=expected_status) if req and req.headers.authorization.is_absent and not request_has_token: for param in [u'error', u'error_description', u'error_uri']: if param in params: resp.complain(1269, param=param)