def test_cache(): decode_gzip = mock.MagicMock() decode_gzip.return_value = b"decoded" encode_gzip = mock.MagicMock() encode_gzip.return_value = b"encoded" with mock.patch.dict(encoding.custom_decode, gzip=decode_gzip): with mock.patch.dict(encoding.custom_encode, gzip=encode_gzip): assert encoding.decode(b"encoded", "gzip") == b"decoded" assert decode_gzip.call_count == 1 # should be cached assert encoding.decode(b"encoded", "gzip") == b"decoded" assert decode_gzip.call_count == 1 # the other way around as well assert encoding.encode(b"decoded", "gzip") == b"encoded" assert encode_gzip.call_count == 0 # different encoding decode_gzip.return_value = b"bar" assert encoding.encode(b"decoded", "deflate") != b"decoded" assert encode_gzip.call_count == 0 # This is not in the cache anymore assert encoding.encode(b"decoded", "gzip") == b"encoded" assert encode_gzip.call_count == 1
def test_encoders_strings(encoder): """ This test is for testing byte->str decoding and str->byte encoding """ assert "" == encoding.decode(b"", encoder) assert "string" == encoding.decode(encoding.encode("string", encoder), encoder) with pytest.raises(TypeError): encoding.encode(b"string", encoder) with pytest.raises(TypeError): encoding.decode("foobar", encoder)
def get_content(self, strict: bool=True) -> bytes: """ The HTTP message body decoded with the content-encoding header (e.g. gzip) Raises: ValueError, when the content-encoding is invalid and strict is True. See also: :py:class:`raw_content`, :py:attr:`text` """ if self.raw_content is None: return None ce = self.headers.get("content-encoding") if ce: try: content = encoding.decode(self.raw_content, ce) # A client may illegally specify a byte -> str encoding here (e.g. utf8) if isinstance(content, str): raise ValueError("Invalid Content-Encoding: {}".format(ce)) return content except ValueError: if strict: raise return self.raw_content else: return self.raw_content
def get_content(self, strict: bool = True) -> bytes: """ The HTTP message body decoded with the content-encoding header (e.g. gzip) Raises: ValueError, when the content-encoding is invalid and strict is True. See also: :py:class:`raw_content`, :py:attr:`text` """ if self.raw_content is None: return None ce = self.headers.get("content-encoding") if ce: try: content = encoding.decode(self.raw_content, ce) # A client may illegally specify a byte -> str encoding here (e.g. utf8) if isinstance(content, str): raise ValueError("Invalid Content-Encoding: {}".format(ce)) return content except ValueError: if strict: raise return self.raw_content else: return self.raw_content
def get_content(self, strict: bool = True) -> Optional[bytes]: """ The uncompressed HTTP message body as bytes. Raises: ValueError, when the HTTP content-encoding is invalid and strict is True. See also: :py:class:`raw_content`, :py:attr:`text` """ if self.raw_content is None: return None ce = self.headers.get("content-encoding") if ce: try: content = encoding.decode(self.raw_content, ce) # A client may illegally specify a byte -> str encoding here (e.g. utf8) if isinstance(content, str): raise ValueError(f"Invalid Content-Encoding: {ce}") return content except ValueError: if strict: raise return self.raw_content else: return self.raw_content
def test_encoders_strings(encoder): """ This test is for testing byte->str decoding and str->byte encoding """ assert "" == encoding.decode(b"", encoder) assert "string" == encoding.decode( encoding.encode( "string", encoder ), encoder ) with pytest.raises(TypeError): encoding.encode(b"string", encoder) with pytest.raises(TypeError): encoding.decode("foobar", encoder)
def test_encoders(encoder): assert "" == encoding.decode("", encoder) assert b"" == encoding.decode(b"", encoder) assert "string" == encoding.decode(encoding.encode("string", encoder), encoder) assert b"string" == encoding.decode(encoding.encode(b"string", encoder), encoder) with tutils.raises(ValueError): encoding.decode(b"foobar", encoder)
def get_text(self, strict: bool = True) -> Optional[str]: """ The uncompressed and decoded HTTP message body as text. Raises: ValueError, when either content-encoding or charset is invalid and strict is True. See also: :py:attr:`content`, :py:class:`raw_content` """ content = self.get_content(strict) if content is None: return None enc = self._guess_encoding(content) try: return cast(str, encoding.decode(content, enc)) except ValueError: if strict: raise return content.decode("utf8", "surrogateescape")
def get_text(self, strict: bool=True) -> Optional[str]: """ The uncompressed and decoded HTTP message body as text. Raises: ValueError, when either content-encoding or charset is invalid and strict is True. See also: :py:attr:`content`, :py:class:`raw_content` """ if self.raw_content is None: return None content = self.get_content(strict) enc = self._guess_encoding(content) try: return encoding.decode(content, enc) except ValueError: if strict: raise return content.decode("utf8", "surrogateescape")
def get_text(self, strict: bool = True) -> Optional[str]: """ The HTTP message body decoded with both content-encoding header (e.g. gzip) and content-type header charset. Raises: ValueError, when either content-encoding or charset is invalid and strict is True. See also: :py:attr:`content`, :py:class:`raw_content` """ if self.raw_content is None: return None enc = self._guess_encoding() content = self.get_content(strict) try: return encoding.decode(content, enc) except ValueError: if strict: raise return content.decode("utf8", "surrogateescape")
def get_text(self, strict: bool=True) -> str: """ The HTTP message body decoded with both content-encoding header (e.g. gzip) and content-type header charset. Raises: ValueError, when either content-encoding or charset is invalid and strict is True. See also: :py:attr:`content`, :py:class:`raw_content` """ if self.raw_content is None: return None enc = self._guess_encoding() content = self.get_content(strict) try: return encoding.decode(content, enc) except ValueError: if strict: raise return content.decode("utf8", "surrogateescape")
def test_encoders(encoder): """ This test is for testing byte->byte encoding/decoding """ assert encoding.decode(None, encoder) is None assert encoding.encode(None, encoder) is None assert b"" == encoding.decode(b"", encoder) assert b"string" == encoding.decode(encoding.encode(b"string", encoder), encoder) with pytest.raises(TypeError): encoding.encode("string", encoder) with pytest.raises(TypeError): encoding.decode("string", encoder) with pytest.raises(ValueError): encoding.decode(b"foobar", encoder)
def get_content(self, strict: bool=True) -> bytes: """ The HTTP message body decoded with the content-encoding header (e.g. gzip) Raises: ValueError, when the content-encoding is invalid and strict is True. See also: :py:class:`raw_content`, :py:attr:`text` """ if self.raw_content is None: return None ce = self.headers.get("content-encoding") if ce: try: return encoding.decode(self.raw_content, ce) except ValueError: if strict: raise return self.raw_content else: return self.raw_content
def test_encoders(encoder): """ This test is for testing byte->byte encoding/decoding """ assert encoding.decode(None, encoder) is None assert encoding.encode(None, encoder) is None assert b"" == encoding.decode(b"", encoder) assert b"string" == encoding.decode( encoding.encode( b"string", encoder ), encoder ) with pytest.raises(TypeError): encoding.encode("string", encoder) with pytest.raises(TypeError): encoding.decode("string", encoder) with pytest.raises(ValueError): encoding.decode(b"foobar", encoder)
def test_encoders(encoder): assert "" == encoding.decode("", encoder) assert b"" == encoding.decode(b"", encoder) assert "string" == encoding.decode( encoding.encode( "string", encoder ), encoder ) assert b"string" == encoding.decode( encoding.encode( b"string", encoder ), encoder ) with tutils.raises(ValueError): encoding.decode(b"foobar", encoder)
def test_identity(encoder): assert b"string" == encoding.decode(b"string", encoder) assert b"string" == encoding.encode(b"string", encoder) with pytest.raises(ValueError): encoding.encode(b"string", "nonexistent encoding")
def dump(self, frame): """ Transform and dump (write / send) a data frame. """ #print('Frame= %s' % frame) requestContentType = None responseContentType = None requestContentEncoding = None responseContentEncoding = None for header in frame["request"]["headers"]: h = header[0].decode('utf-8') #print(h) if h.lower() == "content-type": requestContentType = header[1].decode("utf-8") if h.lower() == "content-encoding": requestContentEncoding = header[1].decode("utf-8") for header in frame["response"]["headers"]: h = header[0].decode('utf-8') #print(h) if h.lower() == "content-type": responseContentType = header[1].decode("utf-8") if h.lower() == "content-encoding": responseContentEncoding = header[1].decode("utf-8") for tfm in self.transformations: for field in tfm['fields']: self.transform_field(frame, field, tfm['func']) #print("requestContentType %s" % requestContentType) #print("responseContentType %s" % responseContentType) #print("requestContentEncoding %s" % requestContentEncoding) #print("responseContentEncoding %s" % responseContentEncoding) if responseContentEncoding: rawContent = frame["response"]["content"] #print(type(rawContent)) #print("rawContent %s " % rawContent) #print("decoding content of type %s" % responseContentEncoding) #print("decoding with input string of type %s" % type(responseContentEncoding)) decodedContent = encoding.decode(rawContent, responseContentEncoding) #print("decodedContent %s" % decodedContent) frame["response"]["content"] = decodedContent if self.storeBinaryContent: if self.isBinaryContent(requestContentType): frame["request"]["content"] = base64.b64encode( frame["request"]["content"]) if self.isBinaryContent(responseContentType): frame["response"]["content"] = base64.b64encode( frame["response"]["content"]) else: if self.isBinaryContent(requestContentType): frame["request"]["content"] = "Binary content removed" if self.isBinaryContent(responseContentType): frame["response"]["content"] = "Binary content removed" frame = self.convert_to_strings(frame) print("Sending frame to Elasticsearch") # If you need to debug this, print/log frame and result as it will show you # what wasc sent and what errors you got back. This generates a lot of noise though... result = requests.post(self.url, json=frame, auth=(self.auth or None)) print(result.text)