def testWriteVarInt64(self):
writer = BitStreamWriter()
writer.writeVarInt64(0)
self.assertEqual(8, writer.getBitPosition())
self.assertEqual(b'\x00', writer.getByteArray())
with self.assertRaises(PythonRuntimeException):
writer.writeVarInt64(-1 << sum(VARINT64_NUM_BITS))
with self.assertRaises(PythonRuntimeException):
writer.writeVarInt64(1 << sum(VARINT64_NUM_BITS))
def testFile(self):
testFilename = "BitStreamTest.bin"
writer = BitStreamWriter()
writer.writeBits(13, 7)
writer.writeString(testFilename)
writer.writeVarInt(-123456)
writer.toFile(testFilename)
reader = BitStreamReader.fromFile(testFilename)
self.assertEqual(13, reader.readBits(7))
self.assertEqual(testFilename, reader.readString())
self.assertEqual(-123456, reader.readVarInt())
def test_file(self):
test_filename = "BitStreamTest.bin"
writer = BitStreamWriter()
writer.write_bits(13, 7)
writer.write_string(test_filename)
writer.write_varint(-123456)
writer.to_file(test_filename)
reader = BitStreamReader.from_file(test_filename)
self.assertEqual(13, reader.read_bits(7))
self.assertEqual(test_filename, reader.read_string())
self.assertEqual(-123456, reader.read_varint())
def _testImpl(self, writeMethod, readMethod, values, maxStartBitPos):
for bitPos in range(maxStartBitPos):
writer = BitStreamWriter()
if bitPos > 0:
writer.writeBits(0, bitPos)
for value in values:
writeMethod(writer, value)
reader = BitStreamReader(buffer=writer.getByteArray())
if bitPos > 0:
reader.readBits(bitPos)
for value in values:
self.assertEqual(value, readMethod(reader), "[bitPos=%d]" % bitPos)
def test_write_varint(self):
writer = BitStreamWriter()
writer.write_varint(0)
self.assertEqual(b'\x00', writer.byte_array)
self.assertEqual(8, writer.bitposition)
writer.write_varint(VARINT_MIN)
self.assertEqual(16, writer.bitposition)
self.assertEqual(b'\x00\x80',
writer.byte_array) # INT64_MIN is encoded as -0
with self.assertRaises(PythonRuntimeException):
writer.write_varint(VARINT_MIN - 1)
with self.assertRaises(PythonRuntimeException):
writer.write_varint(VARINT_MAX + 1)
def _test_bits_impl(self, write_method, read_method, values, numbits):
for bit_pos in range(numbits):
writer = BitStreamWriter()
if bit_pos > 0:
writer.write_bits(0, bit_pos)
for value in values:
write_method(writer, value, numbits)
reader = BitStreamReader(buffer=writer.byte_array)
if bit_pos > 0:
reader.read_bits(bit_pos)
for value in values:
self.assertEqual(value, read_method(reader, numbits),
f"[numbits={numbits}, bit_pos={bit_pos}]")
def _testBitsImpl(self, writeMethod, readMethod, values, numBits):
for bitPos in range(numBits):
writer = BitStreamWriter()
if bitPos > 0:
writer.writeBits(0, bitPos)
for value in values:
writeMethod(writer, value, numBits)
reader = BitStreamReader(buffer=writer.getByteArray())
if bitPos > 0:
reader.readBits(bitPos)
for value in values:
self.assertEqual(value, readMethod(reader, numBits),
"[numBits=%d, bitPos=%d]" % (numBits, bitPos))
def serialize_to_file(obj: typing.Any, filename: str) -> None:
"""
Serializes generated object to the byte buffer.
This is a convenient method for users to easily write given generated object to file.
:param obj: Generated object to serialize.
:param filename: File to write.
:raises PythonRuntimeException: Throws in case of any error during serialization.
"""
writer = BitStreamWriter()
obj.write(writer)
writer.to_file(filename)
def _test_impl(self, write_method, read_method, values, max_start_bit_pos):
for bit_pos in range(max_start_bit_pos):
writer = BitStreamWriter()
if bit_pos > 0:
writer.write_bits(0, bit_pos)
for value in values:
write_method(writer, value)
reader = BitStreamReader(buffer=writer.byte_array)
if bit_pos > 0:
reader.read_bits(bit_pos)
for value in values:
self.assertEqual(value, read_method(reader),
"[bit_pos=%d]" % bit_pos)
def write(self, writer: BitStreamWriter) -> None:
"""
Writes array to the bit stream.
:param writer: Bit stream where to write.
"""
size = len(self._rawArray)
if self._isAuto:
writer.writeVarSize(size)
for index in range(size):
if self._checkOffsetMethod is not None:
writer.alignTo(8)
self._checkOffsetMethod(index, writer.getBitPosition())
self._arrayTraits.write(writer, self._rawArray[index])
def write(self, writer: BitStreamWriter) -> None:
"""
Writes array to the bit stream.
:param writer: Bit stream where to write.
"""
size = len(self._raw_array)
if self._is_auto:
writer.write_varsize(size)
for index in range(size):
if self._check_offset_method is not None:
writer.alignto(8)
self._check_offset_method(index, writer.bitposition)
self._array_traits.write(writer, self._raw_array[index])
def _test_from_reader(self, array_traits, array_values):
array = Array(array_traits, array_values)
writer = BitStreamWriter()
array.write(writer)
reader = BitStreamReader(writer.byte_array)
read_array = Array.from_reader(array_traits, reader, len(array_values))
self.assertEqual(array, read_array)
def test_write_varint16(self):
writer = BitStreamWriter()
writer.write_varint16(0)
self.assertEqual(8, writer.bitposition)
self.assertEqual(b'\x00', writer.byte_array)
with self.assertRaises(PythonRuntimeException):
writer.write_varint16(VARINT16_MIN - 1)
with self.assertRaises(PythonRuntimeException):
writer.write_varint16(VARINT16_MAX + 1)
def _test_read(self, array_traits, array_values):
array = Array(array_traits, array_values)
writer = BitStreamWriter()
array.write(writer)
reader = BitStreamReader(writer.byte_array)
read_array = Array(array_traits)
read_array.read(reader, len(array.raw_array))
self.assertEqual(array, read_array)
auto_array = Array(array_traits, array_values, is_auto=True)
writer = BitStreamWriter()
auto_array.write(writer)
reader = BitStreamReader(writer.byte_array)
read_auto_array = Array(array_traits, is_auto=True)
read_auto_array.read(reader, len(auto_array.raw_array))
self.assertEqual(auto_array, read_auto_array)
aligned_array = Array(
array_traits,
array_values,
check_offset_method=ArrayTest._check_offset_method)
writer = BitStreamWriter()
aligned_array.write(writer)
reader = BitStreamReader(writer.byte_array)
read_aligned_array = Array(
array_traits, check_offset_method=ArrayTest._check_offset_method)
read_aligned_array.read(reader, len(aligned_array.raw_array))
self.assertEqual(aligned_array, read_aligned_array)
if array_traits.HAS_BITSIZEOF_CONSTANT and array_traits.bitsizeof(
) % 8 == 0:
implicit_array = Array(array_traits,
array_values,
is_implicit=True)
writer = BitStreamWriter()
implicit_array.write(writer)
reader = BitStreamReader(writer.byte_array)
read_implicit_array = Array(array_traits, is_implicit=True)
read_implicit_array.read(reader)
self.assertEqual(implicit_array, read_implicit_array)
elif not array_traits.HAS_BITSIZEOF_CONSTANT:
with self.assertRaises(PythonRuntimeException):
Array(array_traits, is_implicit=True).read(reader)
def _test_packed_array_aligned_auto(self, array_traits, array_values,
expected_bitsize):
for i in range(8):
array = Array(array_traits,
array_values,
is_auto=True,
set_offset_method=ArrayTest._set_offset_method,
check_offset_method=ArrayTest._check_offset_method)
bitsize = array.bitsizeof_packed(i)
if expected_bitsize is not None and i == 0:
self.assertEqual(expected_bitsize, bitsize)
self.assertEqual(i + bitsize, array.initialize_offsets_packed(i),
i)
writer = BitStreamWriter()
if i > 0:
writer.write_bits(0, i)
array.write_packed(writer)
self.assertEqual(i + bitsize, writer.bitposition, i)
from_reader = BitStreamReader(writer.byte_array,
writer.bitposition)
self.assertEqual(0, from_reader.read_bits(i))
read_array_from_reader = Array.from_reader_packed(
array_traits,
from_reader,
is_auto=True,
set_offset_method=ArrayTest._set_offset_method,
check_offset_method=ArrayTest._check_offset_method)
self.assertEqual(array, read_array_from_reader, i)
reader = BitStreamReader(writer.byte_array, writer.bitposition)
if i > 0:
self.assertEqual(0, reader.read_bits(i))
read_array = Array(
array_traits,
is_auto=True,
set_offset_method=ArrayTest._set_offset_method,
check_offset_method=ArrayTest._check_offset_method)
read_array.read_packed(reader)
self.assertEqual(array, read_array, i)
def _test_array_implicit(self, array_traits, array_values,
expected_bitsize):
for i in range(8):
array = Array(array_traits, array_values, is_implicit=True)
bitsize = array.bitsizeof(i)
self.assertEqual(expected_bitsize, bitsize, i)
self.assertEqual(i + bitsize, array.initialize_offsets(i), i)
writer = BitStreamWriter()
if i > 0:
writer.write_bits(0, i)
array.write(writer)
self.assertEqual(i + bitsize, writer.bitposition, i)
from_reader = BitStreamReader(writer.byte_array,
writer.bitposition)
if i > 0:
self.assertEqual(0, from_reader.read_bits(i))
if array_traits.HAS_BITSIZEOF_CONSTANT:
read_array_from_reader = Array.from_reader(array_traits,
from_reader,
is_implicit=True)
self.assertEqual(array, read_array_from_reader, i)
else:
with self.assertRaises(PythonRuntimeException):
Array.from_reader(array_traits,
from_reader,
is_implicit=True)
reader = BitStreamReader(writer.byte_array, writer.bitposition)
if i > 0:
self.assertEqual(0, reader.read_bits(i))
read_array = Array(array_traits, is_implicit=True)
if array_traits.HAS_BITSIZEOF_CONSTANT:
read_array.read(reader)
self.assertEqual(array, read_array, i)
else:
with self.assertRaises(PythonRuntimeException):
read_array.read(reader)
def _test_array_aligned(self, array_traits, array_values,
expected_bitsize):
for i in range(8):
array = Array(array_traits,
array_values,
set_offset_method=ArrayTest._set_offset_method,
check_offset_method=ArrayTest._check_offset_method)
bitsize = array.bitsizeof(i)
self.assertEqual(alignto(8, i) - i + expected_bitsize, bitsize, i)
self.assertEqual(i + bitsize, array.initialize_offsets(i), i)
writer = BitStreamWriter()
if i > 0:
writer.write_bits(0, i)
array.write(writer)
self.assertEqual(i + bitsize, writer.bitposition, i)
from_reader = BitStreamReader(writer.byte_array,
writer.bitposition)
self.assertEqual(0, from_reader.read_bits(i))
read_array_from_reader = Array.from_reader(
array_traits,
from_reader,
len(array_values),
set_offset_method=ArrayTest._set_offset_method,
check_offset_method=ArrayTest._check_offset_method)
self.assertEqual(array, read_array_from_reader, i)
reader = BitStreamReader(writer.byte_array, writer.bitposition)
if i > 0:
self.assertEqual(0, reader.read_bits(i))
read_array = Array(
array_traits,
set_offset_method=ArrayTest._set_offset_method,
check_offset_method=ArrayTest._check_offset_method)
read_array.read(reader, len(array_values))
self.assertEqual(array, read_array, i)
def _test_packed_array_implicit(self, array_traits, array_values,
expected_bitsize):
for i in range(8):
array = Array(array_traits, array_values, is_implicit=True)
bitsize = array.bitsizeof_packed(i)
if expected_bitsize is not None:
self.assertEqual(expected_bitsize, bitsize)
self.assertEqual(i + bitsize, array.initialize_offsets_packed(i),
i)
writer = BitStreamWriter()
if i > 0:
writer.write_bits(0, i)
array.write_packed(writer)
self.assertEqual(i + bitsize, writer.bitposition, i)
reader = BitStreamReader(writer.byte_array, writer.bitposition)
if i > 0:
self.assertEqual(0, reader.read_bits(i))
read_array = Array(array_traits, is_implicit=True)
with self.assertRaises(PythonRuntimeException):
read_array.read_packed(reader)
def byte_array(self) -> bytes:
"""
Gets the data which represent the request.
:returns: The request data which are created by serialization of Zserio object.
"""
if not self._is_byte_array_initialized:
writer = BitStreamWriter()
self._zserio_object.write(writer)
self._byte_array = writer.byte_array
self._is_byte_array_initialized = True
return self._byte_array
def testBitPosition(self):
writer = BitStreamWriter()
writer.writeBits(0xaaaa, 16)
self.assertEqual(16, writer.getBitPosition())
writer.writeBits(0xff, 8)
self.assertEqual(24, writer.getBitPosition())
reader = BitStreamReader(buffer=writer.getByteArray())
self.assertEqual(0xaaaa, reader.readBits(16))
self.assertEqual(16, reader.getBitPosition())
reader.setBitPosition(8)
self.assertEqual(8, reader.getBitPosition())
self.assertEqual(0xaaff, reader.readBits(16))
reader.setBitPosition(13)
self.assertEqual(13, reader.getBitPosition())
self.assertEqual(0x02, reader.readBits(3))
self.assertEqual(16, reader.getBitPosition())
self.assertEqual(0xff, reader.readBits(8))
self.assertEqual(24, reader.getBitPosition())
reader.setBitPosition(0)
self.assertEqual(0, reader.getBitPosition())
self.assertEqual(0xaaaaff, reader.readBits(24))
def serialize(obj: typing.Any) -> BitBuffer:
"""
Serializes generated object to the bit buffer.
Because serialization to the bit buffer does not have to be byte aligned (divisible by 8), it's possible
that not all bits of the last byte are used. In this case, only most significant bits of the corresponded
size are used.
:param obj: Generated object to serialize.
:returns: Bit buffer which represents generated object in binary format.
:raises PythonRuntimeException: Throws in case of any error during serialization.
"""
writer = BitStreamWriter()
obj.write(writer)
return BitBuffer(writer.byte_array, writer.bitposition)
def _testWrite(self, arrayTraits, arrayValues, expectedBitSize,
expectedAlignedBitSize):
def _checkOffsetMethod(_index, _bitOffset):
pass
array = Array(arrayTraits, arrayValues)
writer = BitStreamWriter()
array.write(writer)
self.assertEqual(expectedBitSize, writer.getBitPosition())
autoArray = Array(arrayTraits, arrayValues, isAuto=True)
writer = BitStreamWriter()
autoArray.write(writer)
self.assertEqual(
getBitSizeOfVarUInt64(len(arrayValues)) + expectedBitSize,
writer.getBitPosition())
alignedArray = Array(arrayTraits,
arrayValues,
checkOffsetMethod=_checkOffsetMethod)
writer = BitStreamWriter()
alignedArray.write(writer)
self.assertEqual(expectedAlignedBitSize, writer.getBitPosition())
def testWriteUnalignedData(self):
# number expected to be written at offset
testValue = 123
for offset in range(65):
writer = BitStreamWriter()
if offset != 0:
writer.writeBits(0, offset)
writer.writeBits(testValue, 8)
# check written value
buffer = writer.getByteArray()
writtenTestValue = buffer[offset // 8] << (offset % 8)
if offset % 8 != 0:
writtenTestValue |= buffer[offset // 8 + 1] >> (8 - (offset % 8))
self.assertEqual(testValue, writtenTestValue, msg=("Offset: " + str(offset)))
def _test_write(self, array_traits, array_values, expected_bitsize,
expected_aligned_bitsize):
array = Array(array_traits, array_values)
writer = BitStreamWriter()
array.write(writer)
self.assertEqual(expected_bitsize, writer.bitposition)
auto_array = Array(array_traits, array_values, is_auto=True)
writer = BitStreamWriter()
auto_array.write(writer)
self.assertEqual(
bitsizeof_varuint64(len(array_values)) + expected_bitsize,
writer.bitposition)
aligned_array = Array(
array_traits,
array_values,
check_offset_method=ArrayTest._check_offset_method)
writer = BitStreamWriter()
writer.write_bool(False)
aligned_array.write(writer)
self.assertEqual(1 + 7 + expected_aligned_bitsize, writer.bitposition)
def test_write_unaligned_data(self):
# number expected to be written at offset
test_value = 123
for offset in range(65):
writer = BitStreamWriter()
if offset != 0:
writer.write_bits(0, offset)
writer.write_bits(test_value, 8)
# check written value
buffer = writer.byte_array
written_test_value = buffer[offset // 8] << (offset % 8)
if offset % 8 != 0:
written_test_value |= buffer[offset // 8 + 1] >> (8 -
(offset % 8))
self.assertEqual(test_value,
written_test_value,
msg=("Offset: " + str(offset)))
def test_bitposition(self):
writer = BitStreamWriter()
writer.write_bits(0xaaaa, 16)
self.assertEqual(16, writer.bitposition)
writer.write_bits(0xff, 8)
self.assertEqual(24, writer.bitposition)
reader = BitStreamReader(buffer=writer.byte_array)
self.assertEqual(0xaaaa, reader.read_bits(16))
self.assertEqual(16, reader.bitposition)
reader.bitposition = 8
self.assertEqual(8, reader.bitposition)
self.assertEqual(0xaaff, reader.read_bits(16))
reader.bitposition = 13
self.assertEqual(13, reader.bitposition)
self.assertEqual(0x02, reader.read_bits(3))
self.assertEqual(16, reader.bitposition)
self.assertEqual(0xff, reader.read_bits(8))
self.assertEqual(24, reader.bitposition)
reader.bitposition = 0
self.assertEqual(0, reader.bitposition)
self.assertEqual(0xaaaaff, reader.read_bits(24))
def _testRead(self, arrayTraits, arrayValues):
def _checkOffsetMethod(_index, _bitOffset):
pass
array = Array(arrayTraits, arrayValues)
writer = BitStreamWriter()
array.write(writer)
reader = BitStreamReader(writer.getByteArray())
readArray = Array(arrayTraits)
readArray.read(reader, len(array.getRawArray()))
self.assertEqual(array, readArray)
autoArray = Array(arrayTraits, arrayValues, isAuto=True)
writer = BitStreamWriter()
autoArray.write(writer)
reader = BitStreamReader(writer.getByteArray())
readAutoArray = Array(arrayTraits, isAuto=True)
readAutoArray.read(reader, len(autoArray.getRawArray()))
self.assertEqual(autoArray, readAutoArray)
alignedArray = Array(arrayTraits,
arrayValues,
checkOffsetMethod=_checkOffsetMethod)
writer = BitStreamWriter()
alignedArray.write(writer)
reader = BitStreamReader(writer.getByteArray())
readAlignedArray = Array(arrayTraits,
checkOffsetMethod=_checkOffsetMethod)
readAlignedArray.read(reader, len(alignedArray.getRawArray()))
self.assertEqual(alignedArray, readAlignedArray)
if arrayTraits.HAS_BITSIZEOF_CONSTANT and arrayTraits.bitSizeOf(
) % 8 == 0:
implicitArray = Array(arrayTraits, arrayValues, isImplicit=True)
writer = BitStreamWriter()
implicitArray.write(writer)
reader = BitStreamReader(writer.getByteArray())
readImplicitArray = Array(arrayTraits, isImplicit=True)
readImplicitArray.read(reader)
self.assertEqual(implicitArray, readImplicitArray)
elif not arrayTraits.HAS_BITSIZEOF_CONSTANT:
with self.assertRaises(PythonRuntimeException):
Array(arrayTraits, isImplicit=True).read(reader)
def testWriteVarInt(self):
writer = BitStreamWriter()
writer.writeVarInt(0)
self.assertEqual(b'\x00', writer.getByteArray())
self.assertEqual(8, writer.getBitPosition())
writer.writeVarInt(-1 << sum(VARINT_NUM_BITS))
self.assertEqual(16, writer.getBitPosition())
self.assertEqual(b'\x00\x80', writer.getByteArray()) # INT64_MIN is encoded as -0
with self.assertRaises(PythonRuntimeException):
writer.writeVarInt((-1 << sum(VARINT_NUM_BITS)) - 1)
with self.assertRaises(PythonRuntimeException):
writer.writeVarInt(1 << sum(VARINT_NUM_BITS))
def test_alignto(self):
writer = BitStreamWriter()
writer.write_bits(5, 3)
writer.alignto(8)
self.assertEqual(8, writer.bitposition)
writer.write_bits(0, 1)
writer.alignto(16)
self.assertEqual(16, writer.bitposition)
writer.write_bits(0xaa, 9)
writer.alignto(32)
self.assertEqual(32, writer.bitposition)
writer.write_bits(0xaca, 13)
writer.alignto(64)
self.assertEqual(64, writer.bitposition)
writer.write_bits(0xcafe, 16)
reader = BitStreamReader(buffer=writer.byte_array)
self.assertEqual(5, reader.read_bits(3))
reader.alignto(8)
self.assertEqual(8, reader.bitposition)
self.assertEqual(0, reader.read_bits(1))
reader.alignto(16)
self.assertEqual(16, reader.bitposition)
self.assertEqual(0xaa, reader.read_bits(9))
reader.alignto(32)
self.assertEqual(32, reader.bitposition)
self.assertEqual(0xaca, reader.read_bits(13))
reader.alignto(64)
self.assertEqual(64, reader.bitposition)
self.assertEqual(0xcafe, reader.read_bits(16))
def testAlignTo(self):
writer = BitStreamWriter()
writer.writeBits(5, 3)
writer.alignTo(8)
self.assertEqual(8, writer.getBitPosition())
writer.writeBits(0, 1)
writer.alignTo(16)
self.assertEqual(16, writer.getBitPosition())
writer.writeBits(0xaa, 9)
writer.alignTo(32)
self.assertEqual(32, writer.getBitPosition())
writer.writeBits(0xaca, 13)
writer.alignTo(64)
self.assertEqual(64, writer.getBitPosition())
writer.writeBits(0xcafe, 16)
reader = BitStreamReader(buffer=writer.getByteArray())
self.assertEqual(5, reader.readBits(3))
reader.alignTo(8)
self.assertEqual(8, reader.getBitPosition())
self.assertEqual(0, reader.readBits(1))
reader.alignTo(16)
self.assertEqual(16, reader.getBitPosition())
self.assertEqual(0xaa, reader.readBits(9))
reader.alignTo(32)
self.assertEqual(32, reader.getBitPosition())
self.assertEqual(0xaca, reader.readBits(13))
reader.alignTo(64)
self.assertEqual(64, reader.getBitPosition())
self.assertEqual(0xcafe, reader.readBits(16))