Python has_lz4 Examples

Example #1

    def decompress(self):
        codec = self.attributes & self.CODEC_MASK
        assert codec in (
            self.CODEC_GZIP,
            self.CODEC_SNAPPY,
            self.CODEC_LZ4,
            self.CODEC_ZSTD,
        )
        if codec == self.CODEC_GZIP:
            assert has_gzip(), "Gzip decompression unsupported"
            raw_bytes = gzip_decode(self.value)
        elif codec == self.CODEC_SNAPPY:
            assert has_snappy(), "Snappy decompression unsupported"
            raw_bytes = snappy_decode(self.value)
        elif codec == self.CODEC_LZ4:
            assert has_lz4(), "LZ4 decompression unsupported"
            if self.magic == 0:
                raw_bytes = lz4_decode_old_kafka(self.value)
            else:
                raw_bytes = lz4_decode(self.value)
        elif codec == self.CODEC_ZSTD:
            assert has_zstd(), "ZSTD decompression unsupported"
            raw_bytes = zstd_decode(self.value)
        else:
            raise Exception("This should be impossible")

        return MessageSet.decode(raw_bytes, bytes_to_read=len(raw_bytes))

Example #2

File: message.py Project: kngenie/kafka-python

    def decompress(self):
        codec = self.attributes & self.CODEC_MASK
        assert codec in (self.CODEC_GZIP, self.CODEC_SNAPPY, self.CODEC_LZ4)
        if codec == self.CODEC_GZIP:
            assert has_gzip(), 'Gzip decompression unsupported'
            raw_bytes = gzip_decode(self.value)
        elif codec == self.CODEC_SNAPPY:
            assert has_snappy(), 'Snappy decompression unsupported'
            raw_bytes = snappy_decode(self.value)
        elif codec == self.CODEC_LZ4:
            assert has_lz4(), 'LZ4 decompression unsupported'
            if self.magic == 0:
                raw_bytes = lz4_decode_old_kafka(self.value)
            else:
                raw_bytes = lz4_decode(self.value)
        else:
            raise Exception('This should be impossible')

        return MessageSet.decode(raw_bytes, bytes_to_read=len(raw_bytes))

Example #3

File: test_codec.py Project: dpkp/kafka-python

def test_snappy_encode_xerial():
    to_ensure = (
        b'\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01'
        b'\x00\x00\x00\x18'
        b'\xac\x02\x14SNAPPY\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00'
        b'\x00\x00\x00\x18'
        b'\xac\x02\x14XERIAL\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00'
    )

    to_test = (b'SNAPPY' * 50) + (b'XERIAL' * 50)

    compressed = snappy_encode(to_test, xerial_compatible=True, xerial_blocksize=300)
    assert compressed == to_ensure


@pytest.mark.skipif(not has_lz4() or platform.python_implementation() == 'PyPy',
                    reason="python-lz4 crashes on old versions of pypy")
def test_lz4():
    for i in range(1000):
        b1 = random_string(100).encode('utf-8')
        b2 = lz4_decode(lz4_encode(b1))
        assert len(b1) == len(b2)
        assert b1 == b2


@pytest.mark.skipif(not has_lz4() or platform.python_implementation() == 'PyPy',
                    reason="python-lz4 crashes on old versions of pypy")
def test_lz4_old():
    for i in range(1000):
        b1 = random_string(100).encode('utf-8')
        b2 = lz4_decode_old_kafka(lz4_encode_old_kafka(b1))

Example #4

File: test_codec.py Project: Abhishek-Dutta/kafka-python

    to_test = header \
        + struct.pack('!i', block_len) + random_snappy \
        + struct.pack('!i', block_len2) + random_snappy2 \

    assert snappy_decode(to_test) == (b'SNAPPY' * 50) + (b'XERIAL' * 50)


@pytest.mark.skipif(not has_snappy(), reason="Snappy not available")
def test_snappy_encode_xerial():
    to_ensure = (
        b'\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01'
        b'\x00\x00\x00\x18'
        b'\xac\x02\x14SNAPPY\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00'
        b'\x00\x00\x00\x18'
        b'\xac\x02\x14XERIAL\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00'
    )

    to_test = (b'SNAPPY' * 50) + (b'XERIAL' * 50)

    compressed = snappy_encode(to_test, xerial_compatible=True, xerial_blocksize=300)
    assert compressed == to_ensure


@pytest.mark.skipif(not has_lz4(), reason="LZ4 not available")
def test_lz4():
    for i in xrange(1000):
        b1 = random_string(100).encode('utf-8')
        b2 = lz4_decode(lz4_encode(b1))
        assert b1 == b2

Example #5

File: test_codec.py Project: emmatony2/repo

        b'\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01'
        b'\x00\x00\x00\x18'
        b'\xac\x02\x14SNAPPY\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00'
        b'\x00\x00\x00\x18'
        b'\xac\x02\x14XERIAL\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00'
    )

    to_test = (b'SNAPPY' * 50) + (b'XERIAL' * 50)

    compressed = snappy_encode(to_test,
                               xerial_compatible=True,
                               xerial_blocksize=300)
    assert compressed == to_ensure


@pytest.mark.skipif(not has_lz4()
                    or platform.python_implementation() == 'PyPy',
                    reason="python-lz4 crashes on old versions of pypy")
def test_lz4():
    for i in xrange(1000):
        b1 = random_string(100).encode('utf-8')
        b2 = lz4_decode(lz4_encode(b1))
        assert len(b1) == len(b2)
        assert b1 == b2


@pytest.mark.skipif(not has_lz4()
                    or platform.python_implementation() == 'PyPy',
                    reason="python-lz4 crashes on old versions of pypy")
def test_lz4_old():
    for i in xrange(1000):

Example #6

def test_snappy_encode_xerial():
    to_ensure = (
        b'\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01'
        b'\x00\x00\x00\x18'
        b'\xac\x02\x14SNAPPY\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00'
        b'\x00\x00\x00\x18'
        b'\xac\x02\x14XERIAL\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00'
    )

    to_test = (b'SNAPPY' * 50) + (b'XERIAL' * 50)

    compressed = snappy_encode(to_test, xerial_compatible=True, xerial_blocksize=300)
    assert compressed == to_ensure


@pytest.mark.skipif(not has_lz4() or platform.python_implementation() == 'PyPy',
                    reason="python-lz4 crashes on old versions of pypy")
def test_lz4():
    for i in range(1000):
        b1 = random_string(100).encode('utf-8')
        b2 = lz4_decode(lz4_encode(b1))
        assert len(b1) == len(b2)
        assert b1 == b2


@pytest.mark.skipif(not has_lz4() or platform.python_implementation() == 'PyPy',
                    reason="python-lz4 crashes on old versions of pypy")
def test_lz4_old():
    for i in range(1000):
        b1 = random_string(100).encode('utf-8')
        b2 = lz4_decode_old_kafka(lz4_encode_old_kafka(b1))

Example #7

File: test_codec.py Project: zhuomingliang/kafka-python

def test_snappy_encode_xerial():
    to_ensure = (
        b'\x82SNAPPY\x00\x00\x00\x00\x01\x00\x00\x00\x01'
        b'\x00\x00\x00\x18'
        b'\xac\x02\x14SNAPPY\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00'
        b'\x00\x00\x00\x18'
        b'\xac\x02\x14XERIAL\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\xfe\x06\x00\x96\x06\x00'
    )

    to_test = (b'SNAPPY' * 50) + (b'XERIAL' * 50)

    compressed = snappy_encode(to_test, xerial_compatible=True, xerial_blocksize=300)
    assert compressed == to_ensure


@pytest.mark.skipif(not has_lz4(), reason="LZ4 not available")
def test_lz4():
    for i in xrange(1000):
        b1 = random_string(100).encode('utf-8')
        b2 = lz4_decode(lz4_encode(b1))
        assert len(b1) == len(b2)
        assert b1 == b2


@pytest.mark.skipif(not has_lz4(), reason="LZ4 not available")
def test_lz4_old():
    for i in xrange(1000):
        b1 = random_string(100).encode('utf-8')
        b2 = lz4_decode_old_kafka(lz4_encode_old_kafka(b1))
        assert len(b1) == len(b2)
        assert b1 == b2