def test_deflate():
assert b"string" == encoding.decode(encoding.encode(b"string", "deflate"),
"deflate")
assert b"string" == encoding.decode(
encoding.encode(b"string", "deflate")[2:-4], "deflate")
with tutils.raises(ValueError):
encoding.decode(b"bogus", "deflate")
def test_identity():
assert b"string" == encoding.decode("identity", b"string")
assert b"string" == encoding.encode("identity", b"string")
assert b"string" == encoding.encode(b"identity", b"string")
assert b"string" == encoding.decode(b"identity", b"string")
assert not encoding.encode("nonexistent", b"string")
assert not encoding.decode("nonexistent encoding", b"string")
def test_cache():
decode_gzip = mock.MagicMock()
decode_gzip.return_value = b"decoded"
encode_gzip = mock.MagicMock()
encode_gzip.return_value = b"encoded"
with mock.patch.dict(encoding.custom_decode, gzip=decode_gzip):
with mock.patch.dict(encoding.custom_encode, gzip=encode_gzip):
assert encoding.decode(b"encoded", "gzip") == b"decoded"
assert decode_gzip.call_count == 1
# should be cached
assert encoding.decode(b"encoded", "gzip") == b"decoded"
assert decode_gzip.call_count == 1
# the other way around as well
assert encoding.encode(b"decoded", "gzip") == b"encoded"
assert encode_gzip.call_count == 0
# different encoding
decode_gzip.return_value = b"bar"
assert encoding.encode(b"decoded", "deflate") != b"decoded"
assert encode_gzip.call_count == 0
# This is not in the cache anymore
assert encoding.encode(b"decoded", "gzip") == b"encoded"
assert encode_gzip.call_count == 1
def test_deflate():
assert b"string" == encoding.decode("deflate",
encoding.encode("deflate", b"string"))
assert b"string" == encoding.decode(
"deflate",
encoding.encode("deflate", b"string")[2:-4])
assert encoding.decode("deflate", b"bogus") is None
def test_deflate():
assert "string" == encoding.decode("deflate",
encoding.encode("deflate", "string"))
assert "string" == encoding.decode(
"deflate",
encoding.encode("deflate", "string")[2:-4])
assert None == encoding.decode("deflate", "bogus")
def test_gzip():
assert "string" == encoding.decode(
"gzip",
encoding.encode(
"gzip",
"string"))
assert None == encoding.decode("gzip", "bogus")
def test_cache():
decode_gzip = mock.MagicMock()
decode_gzip.return_value = b"decoded"
encode_gzip = mock.MagicMock()
encode_gzip.return_value = b"encoded"
with mock.patch.dict(encoding.custom_decode, gzip=decode_gzip):
with mock.patch.dict(encoding.custom_encode, gzip=encode_gzip):
assert encoding.decode(b"encoded", "gzip") == b"decoded"
assert decode_gzip.call_count == 1
# should be cached
assert encoding.decode(b"encoded", "gzip") == b"decoded"
assert decode_gzip.call_count == 1
# the other way around as well
assert encoding.encode(b"decoded", "gzip") == b"encoded"
assert encode_gzip.call_count == 0
# different encoding
decode_gzip.return_value = b"bar"
assert encoding.encode(b"decoded", "deflate") != b"decoded"
assert encode_gzip.call_count == 0
# This is not in the cache anymore
assert encoding.encode(b"decoded", "gzip") == b"encoded"
assert encode_gzip.call_count == 1
def test_gzip():
assert b"string" == encoding.decode(
"gzip",
encoding.encode(
"gzip",
b"string"
)
)
assert encoding.decode("gzip", b"bogus") is None
def test_gzip():
assert b"string" == encoding.decode(
encoding.encode(
b"string",
"gzip"
),
"gzip"
)
with tutils.raises(ValueError):
encoding.decode(b"bogus", "gzip")
def test_brotli():
assert b"string" == encoding.decode(
encoding.encode(
b"string",
"br"
),
"br"
)
with tutils.raises(ValueError):
encoding.decode(b"bogus", "br")
def test_deflate():
assert "string" == encoding.decode(
"deflate",
encoding.encode(
"deflate",
"string"))
assert "string" == encoding.decode(
"deflate",
encoding.encode(
"deflate",
"string")[
2:-
4])
assert None == encoding.decode("deflate", "bogus")
def get_content_view(viewmode, data, **metadata):
"""
Args:
viewmode: the view to use.
data, **metadata: arguments passed to View instance.
Returns:
A (description, content generator) tuple.
In contrast to calling the views directly, text is always safe-to-print unicode.
Raises:
ContentViewException, if the content view threw an error.
"""
msg = []
headers = metadata.get("headers", {})
enc = headers.get("content-encoding")
if enc and enc != "identity":
decoded = encoding.decode(enc, data)
if decoded:
data = decoded
msg.append("[decoded %s]" % enc)
try:
ret = viewmode(data, **metadata)
# Third-party viewers can fail in unexpected ways...
except Exception as e:
six.reraise(ContentViewException, ContentViewException(str(e)),
sys.exc_info()[2])
if not ret:
ret = get("Raw")(data, **metadata)
msg.append("Couldn't parse: falling back to Raw")
else:
msg.append(ret[0])
return " ".join(msg), safe_to_print(ret[1])
def get_text(self, strict=True):
# type: (bool) -> six.text_type
"""
The HTTP message body decoded with both content-encoding header (e.g. gzip)
and content-type header charset.
Raises:
ValueError, when either content-encoding or charset is invalid and strict is True.
See also: :py:attr:`content`, :py:class:`raw_content`
"""
if self.raw_content is None:
return None
enc = self._guess_encoding()
content = self.get_content(strict)
cached = (
self._text_cache.encoded == content and
(self._text_cache.strict or not strict) and
self._text_cache.encoding == enc
)
if not cached:
is_strict = self._content_cache.strict
try:
decoded = encoding.decode(content, enc)
except ValueError:
if strict:
raise
is_strict = False
decoded = self.content.decode("utf8", "replace" if six.PY2 else "surrogateescape")
self._text_cache = CachedDecode(content, enc, is_strict, decoded)
return self._text_cache.decoded
def get_content_view(viewmode, headers, content, limit, is_request):
"""
Returns a (msg, body) tuple.
"""
if not content:
if is_request:
return "No request content (press tab to view response)", ""
else:
return "No content", ""
msg = []
enc = headers.get("content-encoding")
if enc and enc != "identity":
decoded = encoding.decode(enc, content)
if decoded:
content = decoded
msg.append("[decoded %s]" % enc)
try:
ret = viewmode(headers, content, limit)
# Third-party viewers can fail in unexpected ways...
except Exception:
s = traceback.format_exc()
s = "Content viewer failed: \n" + s
signals.add_event(s, "error")
ret = None
if not ret:
ret = get("Raw")(headers, content, limit)
msg.append("Couldn't parse: falling back to Raw")
else:
msg.append(ret[0])
return " ".join(msg), ret[1]
def get_content_view(viewmode, data, **metadata):
"""
Args:
viewmode: the view to use.
data, **metadata: arguments passed to View instance.
Returns:
A (description, content generator) tuple.
In contrast to calling the views directly, text is always safe-to-print unicode.
Raises:
ContentViewException, if the content view threw an error.
"""
msg = []
headers = metadata.get("headers", {})
enc = headers.get("content-encoding")
if enc and enc != "identity":
decoded = encoding.decode(enc, data)
if decoded:
data = decoded
msg.append("[decoded %s]" % enc)
try:
ret = viewmode(data, **metadata)
# Third-party viewers can fail in unexpected ways...
except Exception as e:
six.reraise(exceptions.ContentViewException, exceptions.ContentViewException(str(e)), sys.exc_info()[2])
if not ret:
ret = get("Raw")(data, **metadata)
msg.append("Couldn't parse: falling back to Raw")
else:
msg.append(ret[0])
return " ".join(msg), safe_to_print(ret[1])
def get_content(self, strict=True):
# type: (bool) -> bytes
"""
The HTTP message body decoded with the content-encoding header (e.g. gzip)
Raises:
ValueError, when the content-encoding is invalid and strict is True.
See also: :py:class:`raw_content`, :py:attr:`text`
"""
if self.raw_content is None:
return None
ce = self.headers.get("content-encoding")
cached = (
self._content_cache.encoded == self.raw_content and
(self._content_cache.strict or not strict) and
self._content_cache.encoding == ce
)
if not cached:
is_strict = True
if ce:
try:
decoded = encoding.decode(self.raw_content, ce)
except ValueError:
if strict:
raise
is_strict = False
decoded = self.raw_content
else:
decoded = self.raw_content
self._content_cache = CachedDecode(self.raw_content, ce, is_strict, decoded)
return self._content_cache.decoded
def test_deflate():
assert b"string" == encoding.decode(
"deflate",
encoding.encode(
"deflate",
b"string"
)
)
assert b"string" == encoding.decode(
"deflate",
encoding.encode(
"deflate",
b"string"
)[2:-4]
)
assert encoding.decode("deflate", b"bogus") is None
def get_content_view(viewmode, hdrItems, content, limit, is_request):
"""
Returns a (msg, body) tuple.
"""
if not content:
if is_request:
return "No request content (press tab to view response)", ""
else:
return "No content", ""
msg = []
hdrs = odict.ODictCaseless([list(i) for i in hdrItems])
enc = hdrs.get_first("content-encoding")
if enc and enc != "identity":
decoded = encoding.decode(enc, content)
if decoded:
content = decoded
msg.append("[decoded %s]" % enc)
try:
ret = viewmode(hdrs, content, limit)
# Third-party viewers can fail in unexpected ways...
except Exception:
s = traceback.format_exc()
s = "Content viewer failed: \n" + s
signals.add_event(s, "error")
ret = None
if not ret:
ret = get("Raw")(hdrs, content, limit)
msg.append("Couldn't parse: falling back to Raw")
else:
msg.append(ret[0])
return " ".join(msg), ret[1]
def test_deflate():
assert b"string" == encoding.decode(
encoding.encode(
b"string",
"deflate"
),
"deflate"
)
assert b"string" == encoding.decode(
encoding.encode(
b"string",
"deflate"
)[2:-4],
"deflate"
)
with tutils.raises(ValueError):
encoding.decode(b"bogus", "deflate")
def get_decoded_content(self):
"""
Returns the decoded content based on the current Content-Encoding
header.
Doesn't change the message iteself or its headers.
"""
ce = self.headers.get("content-encoding")
if not self.content or ce not in encoding.ENCODINGS:
return self.content
return encoding.decode(ce, self.content)
def test_encoders(encoder):
assert "" == encoding.decode("", encoder)
assert b"" == encoding.decode(b"", encoder)
assert "string" == encoding.decode(encoding.encode("string", encoder),
encoder)
assert b"string" == encoding.decode(encoding.encode(b"string", encoder),
encoder)
with tutils.raises(ValueError):
encoding.decode(b"foobar", encoder)
def decode(self):
"""
Decodes body based on the current Content-Encoding header, then
removes the header. If there is no Content-Encoding header, no
action is taken.
Returns True if decoding succeeded, False otherwise.
"""
ce = self.headers.get("content-encoding")
if not self.content or ce not in encoding.ENCODINGS:
return False
data = encoding.decode(ce, self.content)
if data is None:
return False
self.content = data
self.headers.pop("content-encoding", None)
return True
def decode(self):
"""
Decodes body based on the current Content-Encoding header, then
removes the header. If there is no Content-Encoding header, no
action is taken.
Returns True if decoding succeeded, False otherwise.
"""
ce = self.headers.get_first("content-encoding")
if not self.body or ce not in encoding.ENCODINGS:
return False
data = encoding.decode(ce, self.body)
if data is None:
return False
self.body = data
del self.headers["content-encoding"]
return True
def decode(self):
"""
Decodes body based on the current Content-Encoding header, then
removes the header. If there is no Content-Encoding header, no
action is taken.
Returns:
True, if decoding succeeded.
False, otherwise.
"""
ce = self.headers.get("content-encoding")
data = encoding.decode(ce, self.content)
if data is None:
return False
self.content = data
self.headers.pop("content-encoding", None)
return True
def get_content(self, strict: bool = True) -> bytes:
"""
The HTTP message body decoded with the content-encoding header (e.g. gzip)
Raises:
ValueError, when the content-encoding is invalid and strict is True.
See also: :py:class:`raw_content`, :py:attr:`text`
"""
if self.raw_content is None:
return None
ce = self.headers.get("content-encoding")
if ce:
try:
return encoding.decode(self.raw_content, ce)
except ValueError:
if strict:
raise
return self.raw_content
else:
return self.raw_content
def get_text(self, strict: bool = True) -> str:
"""
The HTTP message body decoded with both content-encoding header (e.g. gzip)
and content-type header charset.
Raises:
ValueError, when either content-encoding or charset is invalid and strict is True.
See also: :py:attr:`content`, :py:class:`raw_content`
"""
if self.raw_content is None:
return None
enc = self._guess_encoding()
content = self.get_content(strict)
try:
return encoding.decode(content, enc)
except ValueError:
if strict:
raise
return content.decode("utf8", "surrogateescape")
def get_text(self, strict=True):
# type: (bool) -> six.text_type
"""
The HTTP message body decoded with both content-encoding header (e.g. gzip)
and content-type header charset.
Raises:
ValueError, when either content-encoding or charset is invalid and strict is True.
See also: :py:attr:`content`, :py:class:`raw_content`
"""
if self.raw_content is None:
return None
enc = self._guess_encoding()
content = self.get_content(strict)
try:
return encoding.decode(content, enc)
except ValueError:
if strict:
raise
return content.decode("utf8", "replace" if six.PY2 else "surrogateescape")
def get_content(self, strict=True):
# type: (bool) -> bytes
"""
The HTTP message body decoded with the content-encoding header (e.g. gzip)
Raises:
ValueError, when the content-encoding is invalid and strict is True.
See also: :py:class:`raw_content`, :py:attr:`text`
"""
if self.raw_content is None:
return None
ce = self.headers.get("content-encoding")
if ce:
try:
return encoding.decode(self.raw_content, ce)
except ValueError:
if strict:
raise
return self.raw_content
else:
return self.raw_content
def test_encoders(encoder):
assert "" == encoding.decode("", encoder)
assert b"" == encoding.decode(b"", encoder)
assert "string" == encoding.decode(
encoding.encode(
"string",
encoder
),
encoder
)
assert b"string" == encoding.decode(
encoding.encode(
b"string",
encoder
),
encoder
)
with tutils.raises(ValueError):
encoding.decode(b"foobar", encoder)
def test_identity():
assert "string" == encoding.decode("identity", "string")
assert "string" == encoding.encode("identity", "string")
assert not encoding.encode("nonexistent", "string")
assert None == encoding.decode("nonexistent encoding", "string")
def test_gzip():
assert b"string" == encoding.decode(encoding.encode(b"string", "gzip"),
"gzip")
with tutils.raises(ValueError):
encoding.decode(b"bogus", "gzip")
def test_brotli():
assert b"string" == encoding.decode(encoding.encode(b"string", "br"), "br")
with tutils.raises(ValueError):
encoding.decode(b"bogus", "br")
def test_identity():
assert b"string" == encoding.decode(b"string", "identity")
assert b"string" == encoding.encode(b"string", "identity")
with tutils.raises(ValueError):
encoding.encode(b"string", "nonexistent encoding")
def test_identity():
assert b"string" == encoding.decode(b"string", "identity")
assert b"string" == encoding.encode(b"string", "identity")
with tutils.raises(ValueError):
encoding.encode(b"string", "nonexistent encoding")
def test_gzip():
assert b"string" == encoding.decode("gzip",
encoding.encode("gzip", b"string"))
assert encoding.decode("gzip", b"bogus") is None
def test_gzip():
assert "string" == encoding.decode("gzip",
encoding.encode("gzip", "string"))
assert None == encoding.decode("gzip", "bogus")
