def stream_out(self, data):
"""
Function that compresses source data, sets a partition key then streams out
"""
cdata = Compression.encode_snappy(data,
xerial_compatible=True,
xerial_blocksize=32768)
bcount = str(self.count).encode()
self.count += 1
self.producer.produce(cdata, partition_key=bcount)
def test_snappy_xerial(self):
encoded = compression.encode_snappy(self.text, xerial_compatible=True)
self.assertNotEqual(self.text, encoded)
decoded = compression.decode_snappy(encoded)
self.assertEqual(self.text, decoded)
def test_snappy(self):
encoded = compression.encode_snappy(self.text)
self.assertNotEqual(self.text, encoded)
decoded = compression.decode_snappy(encoded)
self.assertEqual(self.text, decoded)
def test_snappy_large_payload(self):
payload = b''.join([uuid4().bytes for i in range(10)])
c = compression.encode_snappy(payload)
self.assertEqual(compression.decode_snappy(c), payload)
def test_snappy_xerial(self):
encoded = compression.encode_snappy(self.text, xerial_compatible=True)
self.assertNotEqual(self.text, encoded)
decoded = compression.decode_snappy(encoded)
self.assertEqual(self.text, decoded)
def test_snappy(self):
encoded = compression.encode_snappy(self.text)
self.assertNotEqual(self.text, encoded)
decoded = compression.decode_snappy(encoded)
self.assertEqual(self.text, decoded)
def test_snappy_large_payload(self):
payload = b''.join([uuid4().bytes for i in range(10)])
c = compression.encode_snappy(payload)
self.assertEqual(compression.decode_snappy(c), payload)
def test_snappy_large_payload(self):
if platform.python_implementation() == "PyPy":
pytest.skip("PyPy fails to compress large messages with Snappy")
payload = b''.join([uuid4().bytes for i in range(10)])
c = compression.encode_snappy(payload)
self.assertEqual(compression.decode_snappy(c), payload)
