Пример #1
0
 def _maybe_compress(self):
     if self._compression_type:
         buf = self._buffer
         if self._compression_type == self.CODEC_GZIP:
             compressed = gzip_encode(buf)
         elif self._compression_type == self.CODEC_SNAPPY:
             compressed = snappy_encode(buf)
         elif self._compression_type == self.CODEC_LZ4:
             if self._magic == 0:
                 compressed = lz4_encode_old_kafka(bytes(buf))
             else:
                 compressed = lz4_encode(bytes(buf))
         compressed_size = len(compressed)
         size = self._size_in_bytes(key_size=0, value_size=compressed_size)
         if size > len(self._buffer):
             self._buffer = bytearray(size)
         else:
             del self._buffer[size:]
         self._encode_msg(self._buffer,
                          offset=0,
                          timestamp=0,
                          key_size=0,
                          key=None,
                          value_size=compressed_size,
                          value=compressed,
                          attributes=self._compression_type)
         self._pos = size
         return True
     return False
Пример #2
0
 def _maybe_compress(self):
     if self._compression_type:
         buf = self._buffer
         if self._compression_type == self.CODEC_GZIP:
             compressed = gzip_encode(buf)
         elif self._compression_type == self.CODEC_SNAPPY:
             compressed = snappy_encode(buf)
         elif self._compression_type == self.CODEC_LZ4:
             if self._magic == 0:
                 compressed = lz4_encode_old_kafka(bytes(buf))
             else:
                 compressed = lz4_encode(bytes(buf))
         compressed_size = len(compressed)
         size = self._size_in_bytes(key_size=0, value_size=compressed_size)
         if size > len(self._buffer):
             self._buffer = bytearray(size)
         else:
             del self._buffer[size:]
         self._encode_msg(
             self._buffer,
             offset=0, timestamp=0, key_size=0, key=None,
             value_size=compressed_size, value=compressed,
             attributes=self._compression_type)
         self._pos = size
         return True
     return False
Пример #3
0
 def _maybe_compress(self):
     if self._compression_type:
         self._assert_has_codec(self._compression_type)
         data = bytes(self._buffer)
         if self._compression_type == self.CODEC_GZIP:
             compressed = gzip_encode(data)
         elif self._compression_type == self.CODEC_SNAPPY:
             compressed = snappy_encode(data)
         elif self._compression_type == self.CODEC_LZ4:
             if self._magic == 0:
                 compressed = lz4_encode_old_kafka(data)
             else:
                 compressed = lz4_encode(data)
         size = self.size_in_bytes(0,
                                   timestamp=0,
                                   key=None,
                                   value=compressed)
         # We will try to reuse the same buffer if we have enough space
         if size > len(self._buffer):
             self._buffer = bytearray(size)
         else:
             del self._buffer[size:]
         self._encode_msg(start_pos=0,
                          offset=0,
                          timestamp=0,
                          key=None,
                          value=compressed,
                          attributes=self._compression_type)
         return True
     return False
 def _maybe_compress(self):
     if self._compression_type != self.CODEC_NONE:
         self._assert_has_codec(self._compression_type)
         header_size = self.HEADER_STRUCT.size
         data = bytes(self._buffer[header_size:])
         if self._compression_type == self.CODEC_GZIP:
             compressed = gzip_encode(data)
         elif self._compression_type == self.CODEC_SNAPPY:
             compressed = snappy_encode(data)
         elif self._compression_type == self.CODEC_LZ4:
             compressed = lz4_encode(data)
         elif self._compression_type == self.CODEC_ZSTD:
             compressed = zstd_encode(data)
         compressed_size = len(compressed)
         if len(data) <= compressed_size:
             # We did not get any benefit from compression, lets send
             # uncompressed
             return False
         else:
             # Trim bytearray to the required size
             needed_size = header_size + compressed_size
             del self._buffer[needed_size:]
             self._buffer[header_size:needed_size] = compressed
             return True
     return False
Пример #5
0
 def _maybe_compress(self):
     if self._compression_type:
         self._assert_has_codec(self._compression_type)
         data = bytes(self._buffer)
         if self._compression_type == self.CODEC_GZIP:
             compressed = gzip_encode(data)
         elif self._compression_type == self.CODEC_SNAPPY:
             compressed = snappy_encode(data)
         elif self._compression_type == self.CODEC_LZ4:
             if self._magic == 0:
                 compressed = lz4_encode_old_kafka(data)
             else:
                 compressed = lz4_encode(data)
         size = self.size_in_bytes(
             0, timestamp=0, key=None, value=compressed)
         # We will try to reuse the same buffer if we have enough space
         if size > len(self._buffer):
             self._buffer = bytearray(size)
         else:
             del self._buffer[size:]
         self._encode_msg(
             start_pos=0,
             offset=0, timestamp=0, key=None, value=compressed,
             attributes=self._compression_type)
         return True
     return False
Пример #6
0
def test_lz4_incremental():
    for i in range(1000):
        # lz4 max single block size is 4MB
        # make sure we test with multiple-blocks
        b1 = random_string(100).encode('utf-8') * 50000
        b2 = lz4_decode(lz4_encode(b1))
        assert len(b1) == len(b2)
        assert b1 == b2
Пример #7
0
def test_lz4_incremental():
    for i in xrange(1000):
        # lz4 max single block size is 4MB
        # make sure we test with multiple-blocks
        b1 = random_string(100).encode('utf-8') * 50000
        b2 = lz4_decode(lz4_encode(b1))
        assert len(b1) == len(b2)
        assert b1 == b2
Пример #8
0
 def _maybe_compress(self):
     if self._compression_type != self.CODEC_NONE:
         header_size = self.HEADER_STRUCT.size
         data = bytes(self._buffer[header_size:])
         if self._compression_type == self.CODEC_GZIP:
             compressed = gzip_encode(data)
         elif self._compression_type == self.CODEC_SNAPPY:
             compressed = snappy_encode(data)
         elif self._compression_type == self.CODEC_LZ4:
             compressed = lz4_encode(data)
         compressed_size = len(compressed)
         if len(data) <= compressed_size:
             # We did not get any benefit from compression, lets send
             # uncompressed
             return False
         else:
             # Trim bytearray to the required size
             needed_size = header_size + compressed_size
             del self._buffer[needed_size:]
             self._buffer[header_size:needed_size] = compressed
             return True
     return False
Пример #9
0
def test_lz4():
    for i in range(1000):
        b1 = random_string(100).encode('utf-8')
        b2 = lz4_decode(lz4_encode(b1))
        assert len(b1) == len(b2)
        assert b1 == b2
Пример #10
0
def test_lz4():
    for i in xrange(1000):
        b1 = random_string(100).encode('utf-8')
        b2 = lz4_decode(lz4_encode(b1))
        assert len(b1) == len(b2)
        assert b1 == b2