def sync(self): if self._last_sync != self._stream.getPos(): self._stream.writeInt(SYNC_ESCAPE) self._stream.write(self._sync) self._last_sync = self._stream.getPos() if self._block_compress and self._block: def _writeBuffer(data_buf): buf = self._codec.compress(data_buf.toByteArray()) writeVInt(self._stream, len(buf)) self._stream.write(buf) records, keys_len, keys, values_len, values = self._block writeVInt(self._stream, records) _writeBuffer(keys_len) _writeBuffer(keys) _writeBuffer(values_len) _writeBuffer(values) self._block = None
def appendRaw(self, key, value): if self._block_compress: if self._block: records, keys_len, keys, values_len, values = self._block else: keys_len = DataOutputBuffer() keys = DataOutputBuffer() values_len = DataOutputBuffer() values = DataOutputBuffer() records = 0 writeVInt(keys_len, len(key)) keys.write(key) writeVInt(values_len, len(value)) values.write(value) records += 1 self._block = (records, keys_len, keys, values_len, values) current_block_size = keys.getSize() + values.getSize() if current_block_size >= self.COMPRESSION_BLOCK_SIZE: self.sync() else: if self._compress: value = self._codec.compress(value) key_length = len(key) value_length = len(value) self._checkAndWriteSync() self._stream.writeInt(key_length + value_length) self._stream.writeInt(key_length) self._stream.write(key) self._stream.write(value)
def _writeBuffer(data_buf): buf = self._codec.compress(data_buf.toByteArray()) writeVInt(self._stream, len(buf)) self._stream.write(buf)
def write(self, data_output): writeVInt(data_output, self._value)
def writeString(data_output, bytes): bytes = Text.encode(bytes) writeVInt(data_output, len(bytes)) data_output.write(bytes)
def write(self, data_output): writeVInt(data_output, self._length) data_output.write(self._bytes)