def decompress(self): codec = self.attributes & self.CODEC_MASK assert codec in ( self.CODEC_GZIP, self.CODEC_SNAPPY, self.CODEC_LZ4, self.CODEC_ZSTD, ) if codec == self.CODEC_GZIP: assert has_gzip(), "Gzip decompression unsupported" raw_bytes = gzip_decode(self.value) elif codec == self.CODEC_SNAPPY: assert has_snappy(), "Snappy decompression unsupported" raw_bytes = snappy_decode(self.value) elif codec == self.CODEC_LZ4: assert has_lz4(), "LZ4 decompression unsupported" if self.magic == 0: raw_bytes = lz4_decode_old_kafka(self.value) else: raw_bytes = lz4_decode(self.value) elif codec == self.CODEC_ZSTD: assert has_zstd(), "ZSTD decompression unsupported" raw_bytes = zstd_decode(self.value) else: raise Exception("This should be impossible") return MessageSet.decode(raw_bytes, bytes_to_read=len(raw_bytes))
def decompress(self): codec = self.attributes & self.CODEC_MASK assert codec in (self.CODEC_GZIP, self.CODEC_SNAPPY, self.CODEC_LZ4) if codec == self.CODEC_GZIP: assert has_gzip(), 'Gzip decompression unsupported' raw_bytes = gzip_decode(self.value) elif codec == self.CODEC_SNAPPY: assert has_snappy(), 'Snappy decompression unsupported' raw_bytes = snappy_decode(self.value) elif codec == self.CODEC_LZ4: assert has_lz4(), 'LZ4 decompression unsupported' if self.magic == 0: raw_bytes = lz4_decode_old_kafka(self.value) else: raw_bytes = lz4_decode(self.value) else: raise Exception('This should be impossible') return MessageSet.decode(raw_bytes, bytes_to_read=len(raw_bytes))
def test_snappy_encode_xerial(): to_ensure = ( b'\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01' b'\x00\x00\x00\x18' b'\xac\x02\x14SNAPPY\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00' b'\x00\x00\x00\x18' b'\xac\x02\x14XERIAL\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00' ) to_test = (b'SNAPPY' * 50) + (b'XERIAL' * 50) compressed = snappy_encode(to_test, xerial_compatible=True, xerial_blocksize=300) assert compressed == to_ensure @pytest.mark.skipif(not has_lz4() or platform.python_implementation() == 'PyPy', reason="python-lz4 crashes on old versions of pypy") def test_lz4(): for i in range(1000): b1 = random_string(100).encode('utf-8') b2 = lz4_decode(lz4_encode(b1)) assert len(b1) == len(b2) assert b1 == b2 @pytest.mark.skipif(not has_lz4() or platform.python_implementation() == 'PyPy', reason="python-lz4 crashes on old versions of pypy") def test_lz4_old(): for i in range(1000): b1 = random_string(100).encode('utf-8') b2 = lz4_decode_old_kafka(lz4_encode_old_kafka(b1))
to_test = header \ + struct.pack('!i', block_len) + random_snappy \ + struct.pack('!i', block_len2) + random_snappy2 \ assert snappy_decode(to_test) == (b'SNAPPY' * 50) + (b'XERIAL' * 50) @pytest.mark.skipif(not has_snappy(), reason="Snappy not available") def test_snappy_encode_xerial(): to_ensure = ( b'\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01' b'\x00\x00\x00\x18' b'\xac\x02\x14SNAPPY\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00' b'\x00\x00\x00\x18' b'\xac\x02\x14XERIAL\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00' ) to_test = (b'SNAPPY' * 50) + (b'XERIAL' * 50) compressed = snappy_encode(to_test, xerial_compatible=True, xerial_blocksize=300) assert compressed == to_ensure @pytest.mark.skipif(not has_lz4(), reason="LZ4 not available") def test_lz4(): for i in xrange(1000): b1 = random_string(100).encode('utf-8') b2 = lz4_decode(lz4_encode(b1)) assert b1 == b2
b'\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01' b'\x00\x00\x00\x18' b'\xac\x02\x14SNAPPY\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00' b'\x00\x00\x00\x18' b'\xac\x02\x14XERIAL\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00' ) to_test = (b'SNAPPY' * 50) + (b'XERIAL' * 50) compressed = snappy_encode(to_test, xerial_compatible=True, xerial_blocksize=300) assert compressed == to_ensure @pytest.mark.skipif(not has_lz4() or platform.python_implementation() == 'PyPy', reason="python-lz4 crashes on old versions of pypy") def test_lz4(): for i in xrange(1000): b1 = random_string(100).encode('utf-8') b2 = lz4_decode(lz4_encode(b1)) assert len(b1) == len(b2) assert b1 == b2 @pytest.mark.skipif(not has_lz4() or platform.python_implementation() == 'PyPy', reason="python-lz4 crashes on old versions of pypy") def test_lz4_old(): for i in xrange(1000):
def test_snappy_encode_xerial(): to_ensure = ( b'\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01' b'\x00\x00\x00\x18' b'\xac\x02\x14SNAPPY\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00' b'\x00\x00\x00\x18' b'\xac\x02\x14XERIAL\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00' ) to_test = (b'SNAPPY' * 50) + (b'XERIAL' * 50) compressed = snappy_encode(to_test, xerial_compatible=True, xerial_blocksize=300) assert compressed == to_ensure @pytest.mark.skipif(not has_lz4(), reason="LZ4 not available") def test_lz4(): for i in xrange(1000): b1 = random_string(100).encode('utf-8') b2 = lz4_decode(lz4_encode(b1)) assert len(b1) == len(b2) assert b1 == b2 @pytest.mark.skipif(not has_lz4(), reason="LZ4 not available") def test_lz4_old(): for i in xrange(1000): b1 = random_string(100).encode('utf-8') b2 = lz4_decode_old_kafka(lz4_encode_old_kafka(b1)) assert len(b1) == len(b2) assert b1 == b2