def test_reader_corrupt_record_v0_v1(magic):
buffer = _make_compressed_batch(magic)
len_offset = 8
# If the wrapper of compressed messages has a key it will just be ignored.
key_offset = 26 if magic else 18
new_buffer = (
buffer[:key_offset] +
b"\x00\x00\x00\x03123" + # Insert some KEY into wrapper
buffer[key_offset + 4:] # Ignore the 4 byte -1 value for old KEY==None
)
struct.pack_into(">i", new_buffer, len_offset, len(new_buffer) - 12)
batch = LegacyRecordBatch(new_buffer, magic)
msgs = list(batch)
for offset, msg in enumerate(msgs):
assert msg.offset == offset
assert msg.timestamp == (9999999 if magic else None)
assert msg.timestamp_type == (0 if magic else None)
assert msg.key == b"test"
assert msg.value == b"Super"
assert msg.checksum == (-2095076219 if magic else 278251978) & \
0xffffffff
# If the wrapper does not contain a `value` it's corrupted
value_offset = 30 if magic else 22
new_buffer = (
buffer[:value_offset] +
b"\xff\xff\xff\xff" # Set `value` to None by altering size to -1
)
struct.pack_into(">i", new_buffer, len_offset, len(new_buffer) - 12)
with pytest.raises(
CorruptRecordException,
match="Value of compressed message is None"):
batch = LegacyRecordBatch(new_buffer, magic)
list(batch)
def test_read_write_serde_v0_v1_with_compression(compression_type, magic):
builder = LegacyRecordBatchBuilder(
magic=magic, compression_type=compression_type, batch_size=1024 * 1024)
for offset in range(10):
builder.append(
offset, timestamp=9999999, key=b"test", value=b"Super")
buffer = builder.build()
# Broker will set the offset to a proper last offset value
struct.pack_into(">q", buffer, 0, 9)
batch = LegacyRecordBatch(buffer, magic)
assert batch.validate_crc()
assert batch.is_control_batch is False
assert batch.is_transactional is False
assert batch.producer_id is None
assert batch.next_offset == 10
msgs = list(batch)
for offset, msg in enumerate(msgs):
assert msg.offset == offset
assert msg.timestamp == (9999999 if magic else None)
assert msg.timestamp_type == (0 if magic else None)
assert msg.key == b"test"
assert msg.value == b"Super"
assert msg.checksum == (-2095076219 if magic else 278251978) & \
0xffffffff
def test_read_write_serde_v0_v1_with_compression(compression_type, magic):
builder = LegacyRecordBatchBuilder(
magic=magic, compression_type=compression_type, batch_size=1024 * 1024)
for offset in range(10):
builder.append(
offset, timestamp=9999999, key=b"test", value=b"Super")
buffer = builder.build()
# Broker will set the offset to a proper last offset value
struct.pack_into(">q", buffer, 0, 9)
batch = LegacyRecordBatch(buffer, magic)
assert batch.validate_crc()
assert batch.is_control_batch is False
assert batch.is_transactional is False
assert batch.producer_id is None
assert batch.next_offset == 10
msgs = list(batch)
for offset, msg in enumerate(msgs):
assert msg.offset == offset
assert msg.timestamp == (9999999 if magic else None)
assert msg.timestamp_type == (0 if magic else None)
assert msg.key == b"test"
assert msg.value == b"Super"
assert msg.checksum == (-2095076219 if magic else 278251978) & \
0xffffffff
def test_read_write_serde_v0_v1_no_compression(magic, key, value, checksum):
builder = LegacyRecordBatchBuilder(magic=magic,
compression_type=0,
batch_size=1024 * 1024)
builder.append(0, timestamp=9999999, key=key, value=value)
buffer = builder.build()
batch = LegacyRecordBatch(buffer, magic)
assert batch.validate_crc()
assert batch.is_control_batch is False
assert batch.is_transactional is False
assert batch.producer_id is None
assert batch.next_offset == 1
msgs = list(batch)
assert len(msgs) == 1
msg = msgs[0]
assert msg.offset == 0
assert msg.timestamp == (9999999 if magic else None)
assert msg.timestamp_type == (0 if magic else None)
assert msg.key == key
assert msg.value == value
assert msg.checksum == checksum[magic] & 0xffffffff
def test_read_write_serde_v0_v1_with_compression(compression_type, magic):
builder = LegacyRecordBatchBuilder(
magic=magic, compression_type=compression_type, batch_size=1024 * 1024)
for offset in range(10):
builder.append(
offset, timestamp=9999999, key=b"test", value=b"Super")
buffer = builder.build()
batch = LegacyRecordBatch(buffer, magic)
assert batch.validate_crc()
msgs = list(batch)
for offset, msg in enumerate(msgs):
assert msg.offset == offset
assert msg.timestamp == (9999999 if magic else None)
assert msg.timestamp_type == (0 if magic else None)
assert msg.key == b"test"
assert msg.value == b"Super"
assert msg.checksum == (-2095076219 if magic else 278251978) & \
0xffffffff
def test_read_write_serde_v0_v1_no_compression(magic, key, value, checksum):
builder = LegacyRecordBatchBuilder(
magic=magic, compression_type=0, batch_size=1024 * 1024)
builder.append(0, timestamp=9999999, key=key, value=value)
buffer = builder.build()
batch = LegacyRecordBatch(buffer, magic)
assert batch.validate_crc()
assert batch.is_control_batch is False
assert batch.is_transactional is False
assert batch.producer_id is None
assert batch.next_offset == 1
msgs = list(batch)
assert len(msgs) == 1
msg = msgs[0]
assert msg.offset == 0
assert msg.timestamp == (9999999 if magic else None)
assert msg.timestamp_type == (0 if magic else None)
assert msg.key == key
assert msg.value == value
assert msg.checksum == checksum[magic] & 0xffffffff
def test_read_log_append_time_v1():
buffer = _make_compressed_batch(1)
# As Builder does not support creating data with `timestamp_type==1` we
# patch the result manually
buffer[ATTRIBUTES_OFFSET] |= TIMESTAMP_TYPE_MASK
expected_timestamp = 10000000
struct.pack_into(">q", buffer, TIMESTAMP_OFFSET, expected_timestamp)
batch = LegacyRecordBatch(buffer, 1)
msgs = list(batch)
for offset, msg in enumerate(msgs):
assert msg.offset == offset
assert msg.timestamp == expected_timestamp
assert msg.timestamp_type == 1
def test_read_write_serde_v0_v1_no_compression(magic, key, value, checksum):
builder = LegacyRecordBatchBuilder(magic=magic,
compression_type=0,
batch_size=1024 * 1024)
builder.append(0, timestamp=9999999, key=key, value=value)
buffer = builder.build()
batch = LegacyRecordBatch(buffer, magic)
msgs = list(batch)
assert len(msgs) == 1
msg = msgs[0]
assert msg.offset == 0
assert msg.timestamp == (9999999 if magic else None)
assert msg.timestamp_type == (0 if magic else None)
assert msg.key == key
assert msg.value == value
assert msg.checksum == checksum[magic] & 0xffffffff
def test_unavailable_codec(compression_type, name, checker_name):
builder = LegacyRecordBatchBuilder(
magic=0, compression_type=compression_type, batch_size=1024)
builder.append(0, timestamp=None, key=None, value=b"M")
correct_buffer = builder.build()
with mock.patch.object(kafka.codec, checker_name) as mocked:
mocked.return_value = False
# Check that builder raises error
builder = LegacyRecordBatchBuilder(
magic=0, compression_type=compression_type, batch_size=1024)
error_msg = "Libraries for {} compression codec not found".format(name)
with pytest.raises(UnsupportedCodecError, match=error_msg):
builder.append(0, timestamp=None, key=None, value=b"M")
builder.build()
# Check that reader raises same error
batch = LegacyRecordBatch(bytes(correct_buffer), 0)
with pytest.raises(UnsupportedCodecError, match=error_msg):
list(batch)