def stream_out(self, data): """ Function that compresses source data, sets a partition key then streams out """ cdata = Compression.encode_snappy(data, xerial_compatible=True, xerial_blocksize=32768) bcount = str(self.count).encode() self.count += 1 self.producer.produce(cdata, partition_key=bcount)
def test_snappy_xerial(self): encoded = compression.encode_snappy(self.text, xerial_compatible=True) self.assertNotEqual(self.text, encoded) decoded = compression.decode_snappy(encoded) self.assertEqual(self.text, decoded)
def test_snappy(self): encoded = compression.encode_snappy(self.text) self.assertNotEqual(self.text, encoded) decoded = compression.decode_snappy(encoded) self.assertEqual(self.text, decoded)
def test_snappy_large_payload(self): payload = b''.join([uuid4().bytes for i in range(10)]) c = compression.encode_snappy(payload) self.assertEqual(compression.decode_snappy(c), payload)
def test_snappy_large_payload(self): if platform.python_implementation() == "PyPy": pytest.skip("PyPy fails to compress large messages with Snappy") payload = b''.join([uuid4().bytes for i in range(10)]) c = compression.encode_snappy(payload) self.assertEqual(compression.decode_snappy(c), payload)