def test_decode_varint(encoded, decoded):
# We add a bit of bytes around just to check position is calculated
# correctly
value, pos = util.decode_varint(
bytearray(b"\x01\xf0" + encoded + b"\xff\x01"), 2)
assert value == decoded
assert pos - 2 == len(encoded)
def test_decode_varint(encoded, decoded):
# We add a bit of bytes around just to check position is calculated
# correctly
value, pos = util.decode_varint(
bytearray(b"\x01\xf0" + encoded + b"\xff\x01"), 2)
assert value == decoded
assert pos - 2 == len(encoded)
def _read_msg(
self,
decode_varint=decode_varint):
# Record =>
# Length => Varint
# Attributes => Int8
# TimestampDelta => Varlong
# OffsetDelta => Varint
# Key => Bytes
# Value => Bytes
# Headers => [HeaderKey HeaderValue]
# HeaderKey => String
# HeaderValue => Bytes
buffer = self._buffer
pos = self._pos
length, pos = decode_varint(buffer, pos)
start_pos = pos
_, pos = decode_varint(buffer, pos) # attrs can be skipped for now
ts_delta, pos = decode_varint(buffer, pos)
if self.timestamp_type == self.LOG_APPEND_TIME:
timestamp = self.max_timestamp
else:
timestamp = self.first_timestamp + ts_delta
offset_delta, pos = decode_varint(buffer, pos)
offset = self.base_offset + offset_delta
key_len, pos = decode_varint(buffer, pos)
if key_len >= 0:
key = bytes(buffer[pos: pos + key_len])
pos += key_len
else:
key = None
value_len, pos = decode_varint(buffer, pos)
if value_len >= 0:
value = bytes(buffer[pos: pos + value_len])
pos += value_len
else:
value = None
header_count, pos = decode_varint(buffer, pos)
if header_count < 0:
raise CorruptRecordException("Found invalid number of record "
"headers {}".format(header_count))
headers = []
while header_count:
# Header key is of type String, that can't be None
h_key_len, pos = decode_varint(buffer, pos)
if h_key_len < 0:
raise CorruptRecordException(
"Invalid negative header key size {}".format(h_key_len))
h_key = buffer[pos: pos + h_key_len].decode("utf-8")
pos += h_key_len
# Value is of type NULLABLE_BYTES, so it can be None
h_value_len, pos = decode_varint(buffer, pos)
if h_value_len >= 0:
h_value = bytes(buffer[pos: pos + h_value_len])
pos += h_value_len
else:
h_value = None
headers.append((h_key, h_value))
header_count -= 1
# validate whether we have read all header bytes in the current record
if pos - start_pos != length:
raise CorruptRecordException(
"Invalid record size: expected to read {} bytes in record "
"payload, but instead read {}".format(length, pos - start_pos))
self._pos = pos
return DefaultRecord(
offset, timestamp, self.timestamp_type, key, value, headers)
def _read_msg(
self,
decode_varint=decode_varint):
# Record =>
# Length => Varint
# Attributes => Int8
# TimestampDelta => Varlong
# OffsetDelta => Varint
# Key => Bytes
# Value => Bytes
# Headers => [HeaderKey HeaderValue]
# HeaderKey => String
# HeaderValue => Bytes
buffer = self._buffer
pos = self._pos
length, pos = decode_varint(buffer, pos)
start_pos = pos
_, pos = decode_varint(buffer, pos) # attrs can be skipped for now
ts_delta, pos = decode_varint(buffer, pos)
if self.timestamp_type == self.LOG_APPEND_TIME:
timestamp = self.max_timestamp
else:
timestamp = self.first_timestamp + ts_delta
offset_delta, pos = decode_varint(buffer, pos)
offset = self.base_offset + offset_delta
key_len, pos = decode_varint(buffer, pos)
if key_len >= 0:
key = bytes(buffer[pos: pos + key_len])
pos += key_len
else:
key = None
value_len, pos = decode_varint(buffer, pos)
if value_len >= 0:
value = bytes(buffer[pos: pos + value_len])
pos += value_len
else:
value = None
header_count, pos = decode_varint(buffer, pos)
if header_count < 0:
raise CorruptRecordException("Found invalid number of record "
"headers {}".format(header_count))
headers = []
while header_count:
# Header key is of type String, that can't be None
h_key_len, pos = decode_varint(buffer, pos)
if h_key_len < 0:
raise CorruptRecordException(
"Invalid negative header key size {}".format(h_key_len))
h_key = buffer[pos: pos + h_key_len].decode("utf-8")
pos += h_key_len
# Value is of type NULLABLE_BYTES, so it can be None
h_value_len, pos = decode_varint(buffer, pos)
if h_value_len >= 0:
h_value = bytes(buffer[pos: pos + h_value_len])
pos += h_value_len
else:
h_value = None
headers.append((h_key, h_value))
header_count -= 1
# validate whether we have read all header bytes in the current record
if pos - start_pos != length:
raise CorruptRecordException(
"Invalid record size: expected to read {} bytes in record "
"payload, but instead read {}".format(length, pos - start_pos))
self._pos = pos
return DefaultRecord(
offset, timestamp, self.timestamp_type, key, value, headers)