def sync(self):
if self._last_sync != self._stream.getPos():
self._stream.writeInt(SYNC_ESCAPE)
self._stream.write(self._sync)
self._last_sync = self._stream.getPos()
if self._block_compress and self._block:
def _writeBuffer(data_buf):
buf = self._codec.compress(data_buf.toByteArray())
writeVInt(self._stream, len(buf))
self._stream.write(buf)
records, keys_len, keys, values_len, values = self._block
writeVInt(self._stream, records)
_writeBuffer(keys_len)
_writeBuffer(keys)
_writeBuffer(values_len)
_writeBuffer(values)
self._block = None
def appendRaw(self, key, value):
if self._block_compress:
if self._block:
records, keys_len, keys, values_len, values = self._block
else:
keys_len = DataOutputBuffer()
keys = DataOutputBuffer()
values_len = DataOutputBuffer()
values = DataOutputBuffer()
records = 0
writeVInt(keys_len, len(key))
keys.write(key)
writeVInt(values_len, len(value))
values.write(value)
records += 1
self._block = (records, keys_len, keys, values_len, values)
current_block_size = keys.getSize() + values.getSize()
if current_block_size >= self.COMPRESSION_BLOCK_SIZE:
self.sync()
else:
if self._compress:
value = self._codec.compress(value)
key_length = len(key)
value_length = len(value)
self._checkAndWriteSync()
self._stream.writeInt(key_length + value_length)
self._stream.writeInt(key_length)
self._stream.write(key)
self._stream.write(value)
def _writeBuffer(data_buf):
buf = self._codec.compress(data_buf.toByteArray())
writeVInt(self._stream, len(buf))
self._stream.write(buf)