def test_cache():
decode_gzip = mock.MagicMock()
decode_gzip.return_value = b"decoded"
encode_gzip = mock.MagicMock()
encode_gzip.return_value = b"encoded"
with mock.patch.dict(encoding.custom_decode, gzip=decode_gzip):
with mock.patch.dict(encoding.custom_encode, gzip=encode_gzip):
assert encoding.decode(b"encoded", "gzip") == b"decoded"
assert decode_gzip.call_count == 1
# should be cached
assert encoding.decode(b"encoded", "gzip") == b"decoded"
assert decode_gzip.call_count == 1
# the other way around as well
assert encoding.encode(b"decoded", "gzip") == b"encoded"
assert encode_gzip.call_count == 0
# different encoding
decode_gzip.return_value = b"bar"
assert encoding.encode(b"decoded", "deflate") != b"decoded"
assert encode_gzip.call_count == 0
# This is not in the cache anymore
assert encoding.encode(b"decoded", "gzip") == b"encoded"
assert encode_gzip.call_count == 1
def test_cache():
decode_gzip = mock.MagicMock()
decode_gzip.return_value = b"decoded"
encode_gzip = mock.MagicMock()
encode_gzip.return_value = b"encoded"
with mock.patch.dict(encoding.custom_decode, gzip=decode_gzip):
with mock.patch.dict(encoding.custom_encode, gzip=encode_gzip):
assert encoding.decode(b"encoded", "gzip") == b"decoded"
assert decode_gzip.call_count == 1
# should be cached
assert encoding.decode(b"encoded", "gzip") == b"decoded"
assert decode_gzip.call_count == 1
# the other way around as well
assert encoding.encode(b"decoded", "gzip") == b"encoded"
assert encode_gzip.call_count == 0
# different encoding
decode_gzip.return_value = b"bar"
assert encoding.encode(b"decoded", "deflate") != b"decoded"
assert encode_gzip.call_count == 0
# This is not in the cache anymore
assert encoding.encode(b"decoded", "gzip") == b"encoded"
assert encode_gzip.call_count == 1
def test_encoders_strings(encoder):
"""
This test is for testing byte->str decoding
and str->byte encoding
"""
assert "" == encoding.decode(b"", encoder)
assert "string" == encoding.decode(encoding.encode("string", encoder),
encoder)
with pytest.raises(TypeError):
encoding.encode(b"string", encoder)
with pytest.raises(TypeError):
encoding.decode("foobar", encoder)
def get_content(self, strict: bool=True) -> bytes:
"""
The HTTP message body decoded with the content-encoding header (e.g. gzip)
Raises:
ValueError, when the content-encoding is invalid and strict is True.
See also: :py:class:`raw_content`, :py:attr:`text`
"""
if self.raw_content is None:
return None
ce = self.headers.get("content-encoding")
if ce:
try:
content = encoding.decode(self.raw_content, ce)
# A client may illegally specify a byte -> str encoding here (e.g. utf8)
if isinstance(content, str):
raise ValueError("Invalid Content-Encoding: {}".format(ce))
return content
except ValueError:
if strict:
raise
return self.raw_content
else:
return self.raw_content
def get_content(self, strict: bool = True) -> bytes:
"""
The HTTP message body decoded with the content-encoding header (e.g. gzip)
Raises:
ValueError, when the content-encoding is invalid and strict is True.
See also: :py:class:`raw_content`, :py:attr:`text`
"""
if self.raw_content is None:
return None
ce = self.headers.get("content-encoding")
if ce:
try:
content = encoding.decode(self.raw_content, ce)
# A client may illegally specify a byte -> str encoding here (e.g. utf8)
if isinstance(content, str):
raise ValueError("Invalid Content-Encoding: {}".format(ce))
return content
except ValueError:
if strict:
raise
return self.raw_content
else:
return self.raw_content
def get_content(self, strict: bool = True) -> Optional[bytes]:
"""
The uncompressed HTTP message body as bytes.
Raises:
ValueError, when the HTTP content-encoding is invalid and strict is True.
See also: :py:class:`raw_content`, :py:attr:`text`
"""
if self.raw_content is None:
return None
ce = self.headers.get("content-encoding")
if ce:
try:
content = encoding.decode(self.raw_content, ce)
# A client may illegally specify a byte -> str encoding here (e.g. utf8)
if isinstance(content, str):
raise ValueError(f"Invalid Content-Encoding: {ce}")
return content
except ValueError:
if strict:
raise
return self.raw_content
else:
return self.raw_content
def test_encoders_strings(encoder):
"""
This test is for testing byte->str decoding
and str->byte encoding
"""
assert "" == encoding.decode(b"", encoder)
assert "string" == encoding.decode(
encoding.encode(
"string",
encoder
),
encoder
)
with pytest.raises(TypeError):
encoding.encode(b"string", encoder)
with pytest.raises(TypeError):
encoding.decode("foobar", encoder)
def test_encoders(encoder):
assert "" == encoding.decode("", encoder)
assert b"" == encoding.decode(b"", encoder)
assert "string" == encoding.decode(encoding.encode("string", encoder),
encoder)
assert b"string" == encoding.decode(encoding.encode(b"string", encoder),
encoder)
with tutils.raises(ValueError):
encoding.decode(b"foobar", encoder)
def get_text(self, strict: bool = True) -> Optional[str]:
"""
The uncompressed and decoded HTTP message body as text.
Raises:
ValueError, when either content-encoding or charset is invalid and strict is True.
See also: :py:attr:`content`, :py:class:`raw_content`
"""
content = self.get_content(strict)
if content is None:
return None
enc = self._guess_encoding(content)
try:
return cast(str, encoding.decode(content, enc))
except ValueError:
if strict:
raise
return content.decode("utf8", "surrogateescape")
def get_text(self, strict: bool=True) -> Optional[str]:
"""
The uncompressed and decoded HTTP message body as text.
Raises:
ValueError, when either content-encoding or charset is invalid and strict is True.
See also: :py:attr:`content`, :py:class:`raw_content`
"""
if self.raw_content is None:
return None
content = self.get_content(strict)
enc = self._guess_encoding(content)
try:
return encoding.decode(content, enc)
except ValueError:
if strict:
raise
return content.decode("utf8", "surrogateescape")
def get_text(self, strict: bool = True) -> Optional[str]:
"""
The HTTP message body decoded with both content-encoding header (e.g. gzip)
and content-type header charset.
Raises:
ValueError, when either content-encoding or charset is invalid and strict is True.
See also: :py:attr:`content`, :py:class:`raw_content`
"""
if self.raw_content is None:
return None
enc = self._guess_encoding()
content = self.get_content(strict)
try:
return encoding.decode(content, enc)
except ValueError:
if strict:
raise
return content.decode("utf8", "surrogateescape")
def get_text(self, strict: bool=True) -> str:
"""
The HTTP message body decoded with both content-encoding header (e.g. gzip)
and content-type header charset.
Raises:
ValueError, when either content-encoding or charset is invalid and strict is True.
See also: :py:attr:`content`, :py:class:`raw_content`
"""
if self.raw_content is None:
return None
enc = self._guess_encoding()
content = self.get_content(strict)
try:
return encoding.decode(content, enc)
except ValueError:
if strict:
raise
return content.decode("utf8", "surrogateescape")
def test_encoders(encoder):
"""
This test is for testing byte->byte encoding/decoding
"""
assert encoding.decode(None, encoder) is None
assert encoding.encode(None, encoder) is None
assert b"" == encoding.decode(b"", encoder)
assert b"string" == encoding.decode(encoding.encode(b"string", encoder),
encoder)
with pytest.raises(TypeError):
encoding.encode("string", encoder)
with pytest.raises(TypeError):
encoding.decode("string", encoder)
with pytest.raises(ValueError):
encoding.decode(b"foobar", encoder)
def get_content(self, strict: bool=True) -> bytes:
"""
The HTTP message body decoded with the content-encoding header (e.g. gzip)
Raises:
ValueError, when the content-encoding is invalid and strict is True.
See also: :py:class:`raw_content`, :py:attr:`text`
"""
if self.raw_content is None:
return None
ce = self.headers.get("content-encoding")
if ce:
try:
return encoding.decode(self.raw_content, ce)
except ValueError:
if strict:
raise
return self.raw_content
else:
return self.raw_content
def test_encoders(encoder):
"""
This test is for testing byte->byte encoding/decoding
"""
assert encoding.decode(None, encoder) is None
assert encoding.encode(None, encoder) is None
assert b"" == encoding.decode(b"", encoder)
assert b"string" == encoding.decode(
encoding.encode(
b"string",
encoder
),
encoder
)
with pytest.raises(TypeError):
encoding.encode("string", encoder)
with pytest.raises(TypeError):
encoding.decode("string", encoder)
with pytest.raises(ValueError):
encoding.decode(b"foobar", encoder)
def test_encoders(encoder):
assert "" == encoding.decode("", encoder)
assert b"" == encoding.decode(b"", encoder)
assert "string" == encoding.decode(
encoding.encode(
"string",
encoder
),
encoder
)
assert b"string" == encoding.decode(
encoding.encode(
b"string",
encoder
),
encoder
)
with tutils.raises(ValueError):
encoding.decode(b"foobar", encoder)
def test_identity(encoder):
assert b"string" == encoding.decode(b"string", encoder)
assert b"string" == encoding.encode(b"string", encoder)
with pytest.raises(ValueError):
encoding.encode(b"string", "nonexistent encoding")
def test_identity(encoder):
assert b"string" == encoding.decode(b"string", encoder)
assert b"string" == encoding.encode(b"string", encoder)
with pytest.raises(ValueError):
encoding.encode(b"string", "nonexistent encoding")
def dump(self, frame):
"""
Transform and dump (write / send) a data frame.
"""
#print('Frame= %s' % frame)
requestContentType = None
responseContentType = None
requestContentEncoding = None
responseContentEncoding = None
for header in frame["request"]["headers"]:
h = header[0].decode('utf-8')
#print(h)
if h.lower() == "content-type":
requestContentType = header[1].decode("utf-8")
if h.lower() == "content-encoding":
requestContentEncoding = header[1].decode("utf-8")
for header in frame["response"]["headers"]:
h = header[0].decode('utf-8')
#print(h)
if h.lower() == "content-type":
responseContentType = header[1].decode("utf-8")
if h.lower() == "content-encoding":
responseContentEncoding = header[1].decode("utf-8")
for tfm in self.transformations:
for field in tfm['fields']:
self.transform_field(frame, field, tfm['func'])
#print("requestContentType %s" % requestContentType)
#print("responseContentType %s" % responseContentType)
#print("requestContentEncoding %s" % requestContentEncoding)
#print("responseContentEncoding %s" % responseContentEncoding)
if responseContentEncoding:
rawContent = frame["response"]["content"]
#print(type(rawContent))
#print("rawContent %s " % rawContent)
#print("decoding content of type %s" % responseContentEncoding)
#print("decoding with input string of type %s" % type(responseContentEncoding))
decodedContent = encoding.decode(rawContent,
responseContentEncoding)
#print("decodedContent %s" % decodedContent)
frame["response"]["content"] = decodedContent
if self.storeBinaryContent:
if self.isBinaryContent(requestContentType):
frame["request"]["content"] = base64.b64encode(
frame["request"]["content"])
if self.isBinaryContent(responseContentType):
frame["response"]["content"] = base64.b64encode(
frame["response"]["content"])
else:
if self.isBinaryContent(requestContentType):
frame["request"]["content"] = "Binary content removed"
if self.isBinaryContent(responseContentType):
frame["response"]["content"] = "Binary content removed"
frame = self.convert_to_strings(frame)
print("Sending frame to Elasticsearch")
# If you need to debug this, print/log frame and result as it will show you
# what wasc sent and what errors you got back. This generates a lot of noise though...
result = requests.post(self.url, json=frame, auth=(self.auth or None))
print(result.text)
