def writeBitBuffer(self, bitBuffer: BitBuffer) -> None:
"""
Writes a bit buffer to the underlying storage. Length of the bit buffer is written as varuint64
at the beginning.
:param bitBuffer: Bit buffer to write.
"""
bitSize = bitBuffer.getBitSize()
self.writeVarSize(bitSize)
writeBuffer = bitBuffer.getBuffer()
numBytesToWrite = bitSize // 8
numRestBits = bitSize - numBytesToWrite * 8
beginBitPosition = self._bitPosition
if (beginBitPosition & 0x07) != 0:
# we are not aligned to byte
for i in range(numBytesToWrite):
self.writeBits(writeBuffer[i], 8)
else:
# we are aligned to byte
self._byteArray += writeBuffer[0:numBytesToWrite]
self._bitPosition += numBytesToWrite * 8
if numRestBits > 0:
self.writeBits(writeBuffer[numBytesToWrite] >> (8 - numRestBits),
numRestBits)
def testBitBuffer(self):
values = [
BitBuffer(bytes([0xAB, 0x07]), 11),
BitBuffer(bytes([0xAB, 0xCD, 0x7F]), 23)
]
self._testImpl(BitStreamWriter.writeBitBuffer, BitStreamReader.readBitBuffer, values, 7)
def test_buffer_constructor(self):
bytesize = 2
bitbuffer = BitBuffer(bytes([1, 2]))
self.assertEqual(8 * bytesize, bitbuffer.bitsize)
empty_bitsize = 0
empty_bitbuffer = BitBuffer(bytes([]))
self.assertEqual(empty_bitsize, empty_bitbuffer.bitsize)
def testReadBitBuffer(self):
reader = BitStreamReader(bytes(b'\x0B\xAB\xE1\xE0\x1F\xC0'))
self.assertEqual(BitBuffer(bytes([0xAB, 0x07]), 11),
reader.readBitBuffer())
self.assertEqual(BitBuffer(bytes([0x00, 0x7F]), 15),
reader.readBitBuffer())
with self.assertRaises(PythonRuntimeException):
reader.readBitBuffer()
def test_deserialize(self):
bitbuffer = BitBuffer(b'\x00\x01\xBD\x5A', 31)
dummy_object = deserialize(DummyObject, bitbuffer, 0xAB)
self.assertEqual(0xDEAD, dummy_object.get_value())
wrong_bitbuffer = BitBuffer(b'\x00\x01\xBD\x5A', 30)
with self.assertRaises(PythonRuntimeException):
deserialize(DummyObject, wrong_bitbuffer, 0xAB) # reading behind the stream!
def testFromBitBuffer(self):
bitBuffer = BitBuffer(bytes([0xAE, 0xEA, 0x80]), 17)
reader = BitStreamReader.fromBitBuffer(bitBuffer)
self.assertEqual(bitBuffer.getBitSize(), reader.getBufferBitSize())
self.assertEqual(0xAEE, reader.readBits(12))
self.assertEqual(0x0A, reader.readBits(4))
self.assertEqual(0x01, reader.readBits(1))
with self.assertRaises(PythonRuntimeException):
reader.readBits(1)
def fromBitBuffer(cls: typing.Type['BitStreamReader'], bitBuffer: BitBuffer) -> 'BitStreamReader':
"""
Constructs bit stream reader from bit buffer.
:param bitBuffer: Bit buffer to read as a bit stream.
"""
instance = cls(bitBuffer.getBuffer())
instance._bitSize = bitBuffer.getBitSize()
return instance
def test_buffer_bitsize_constructor(self):
bitsize = 11
bitbuffer = BitBuffer(bytes([0x01, 0xE0]), bitsize)
self.assertEqual(bitsize, bitbuffer.bitsize)
empty_bitsize = 0
empty_bitbuffer = BitBuffer(bytes([]), empty_bitsize)
self.assertEqual(empty_bitsize, empty_bitbuffer.bitsize)
out_of_range_bitsize = 9
with self.assertRaises(PythonRuntimeException):
BitBuffer([1], out_of_range_bitsize) # throws!
def testBitBufferArray(self):
arrayTraits = BitBufferArrayTraits()
array1Values = [
BitBuffer(bytes([0xAB, 0x07]), 11),
BitBuffer(bytes([0xAB, 0xCD, 0x7F]), 23)
]
array1BitSizeOf = 8 + 11 + 8 + 23
array1AlignedBitSizeOf = 8 + 11 + 5 + 8 + 23
array2Values = [
BitBuffer(bytes([0xBA, 0x07]), 11),
BitBuffer(bytes([0xBA, 0xDC, 0x7F]), 23)
]
self._testArray(arrayTraits, array1Values, array1BitSizeOf,
array1AlignedBitSizeOf, array2Values)
def test_bitbuffer_array(self):
array_traits = BitBufferArrayTraits()
array1_values = [
BitBuffer(bytes([0xAB, 0x07]), 11),
BitBuffer(bytes([0xAB, 0xCD, 0x7F]), 23)
]
array1_bitsizeof = 8 + 11 + 8 + 23
array1_aligned_bitsizeof = 8 + 11 + 5 + 8 + 23
array2_values = [
BitBuffer(bytes([0xBA, 0x07]), 11),
BitBuffer(bytes([0xBA, 0xDC, 0x7F]), 23)
]
self._test_array(array_traits, array1_values, array1_bitsizeof,
array1_aligned_bitsizeof, array2_values)
def test_read_unaligned_data(self):
# number expected to read at offset
test_value = 123
for offset in range(65):
buffer = bytearray((8 + offset + 7) // 8)
# write test value at offset to data buffer
buffer[offset // 8] = test_value >> (offset % 8)
if offset % 8 != 0: # don't write behind the buffer
buffer[offset // 8 + 1] = 0xff & test_value << (8 - offset % 8)
bitbuffer = BitBuffer(buffer, 8 + offset)
reader = BitStreamReader.from_bitbuffer(bitbuffer)
# read offset bits
self.assertEqual(0, reader.read_bits(offset))
# read magic number
self.assertEqual(test_value,
reader.read_bits(8),
msg=("Offset: " + str(offset)))
# check eof
with self.assertRaises(PythonRuntimeException):
reader.read_bits(1)
def readBitBuffer(self):
"""
Reads a bit buffer from the stream.
:return: Read bit buffer.
:raises PythonRuntimeException: If the reading goes behind the stream.
"""
bitSize = self.readVarUInt64()
numBytesToRead = bitSize // 8
numRestBits = bitSize - numBytesToRead * 8
byteSize = (bitSize + 7) // 8
readBuffer = bytearray(byteSize)
beginBitPosition = self._bitPosition
if (beginBitPosition & 0x07) != 0:
# we are not aligned to byte
for i in range(numBytesToRead):
readBuffer[i] = self.readBits(8)
else:
# we are aligned to byte
self.setBitPosition(beginBitPosition + numBytesToRead * 8)
beginBytePosition = beginBitPosition // 8
readBuffer[0:numBytesToRead] = self._buffer[
beginBytePosition:beginBytePosition + numBytesToRead]
if numRestBits != 0:
readBuffer[numBytesToRead] = self.readBits(numRestBits)
return BitBuffer(readBuffer, bitSize)
def read_bitbuffer(self) -> BitBuffer:
"""
Reads a bit buffer from the stream.
:returns: Read bit buffer.
:raises PythonRuntimeException: If the reading goes behind the stream.
"""
bitsize = self.read_varsize()
num_bytes_to_read = bitsize // 8
num_rest_bits = bitsize - num_bytes_to_read * 8
bytesize = (bitsize + 7) // 8
read_buffer = bytearray(bytesize)
begin_bitposition = self._bitposition
if (begin_bitposition & 0x07) != 0:
# we are not aligned to byte
for i in range(num_bytes_to_read):
read_buffer[i] = self.read_bits(8)
else:
# we are aligned to byte
self.bitposition = begin_bitposition + num_bytes_to_read * 8
begin_byte_position = begin_bitposition // 8
buffer = self._bitbuffer.buffer
read_buffer[0:num_bytes_to_read] = buffer[
begin_byte_position:begin_byte_position + num_bytes_to_read]
if num_rest_bits != 0:
read_buffer[num_bytes_to_read] = self.read_bits(num_rest_bits) << (
8 - num_rest_bits)
return BitBuffer(read_buffer, bitsize)
def test_from_bitbuffer(self):
bitbuffer = BitBuffer(bytes([0xAE, 0xEA, 0x80]), 17)
reader = BitStreamReader.from_bitbuffer(bitbuffer)
self.assertEqual(bitbuffer.bitsize, reader.buffer_bitsize)
self.assertEqual(0xAEE, reader.read_bits(12))
self.assertEqual(0x0A, reader.read_bits(4))
self.assertEqual(0x01, reader.read_bits(1))
with self.assertRaises(PythonRuntimeException):
reader.read_bits(1)
def getBitSizeOfBitBuffer(bitBuffer: BitBuffer) -> int:
"""
Gets the bit size of bit buffer which is stored in bit stream.
:param bitBuffer: Bit buffer for calculation.
:returns: Length of bit buffer in bits.
:raises PythonRuntimeException: Throws if given bit buffer is too long.
"""
bitBufferSize = bitBuffer.getBitSize()
# bit buffer consists of varsize for bit size followed by the bits
return getBitSizeOfVarSize(bitBufferSize) + bitBufferSize
def testBufferConstructor(self):
byteSize = 2
bitBuffer = BitBuffer(bytes([1, 2]))
self.assertEqual(8 * byteSize, bitBuffer.getBitSize())
emptyBitSize = 0
emptyBitBuffer = BitBuffer(bytes([]))
self.assertEqual(emptyBitSize, emptyBitBuffer.getBitSize())
def testBufferBitSizeConstructor(self):
bitSize = 11
bitBuffer = BitBuffer(bytes([1, 2]), bitSize)
self.assertEqual(bitSize, bitBuffer.getBitSize())
emptyBitSize = 0
emptyBitBuffer = BitBuffer(bytes([]), emptyBitSize)
self.assertEqual(emptyBitSize, emptyBitBuffer.getBitSize())
outOfRangeBitSize = 9
with self.assertRaises(PythonRuntimeException):
BitBuffer([1], outOfRangeBitSize) # throws!
def serialize(obj: typing.Any) -> BitBuffer:
"""
Serializes generated object to the bit buffer.
Because serialization to the bit buffer does not have to be byte aligned (divisible by 8), it's possible
that not all bits of the last byte are used. In this case, only most significant bits of the corresponded
size are used.
:param obj: Generated object to serialize.
:returns: Bit buffer which represents generated object in binary format.
:raises PythonRuntimeException: Throws in case of any error during serialization.
"""
writer = BitStreamWriter()
obj.write(writer)
return BitBuffer(writer.byte_array, writer.bitposition)
def __init__(self,
buffer: bytes,
bitsize: typing.Optional[int] = None) -> None:
"""
Constructs bit stream reader from bytes buffer.
Because bit buffer size does not have to be byte aligned (divisible by 8), it's possible that not all
bits of the last byte are used. In this case, only most significant bits of the corresponded size are
used.
:param buffer: Bytes-like buffer to read as a bit stream.
:param bitsize: Number of bits stored in buffer to use.
:raises PythonRuntimeException: If bitsize is out of range.
"""
self._bitbuffer: BitBuffer = BitBuffer(buffer, bitsize)
self._bitposition: int = 0
def deserialize_bytes(obj_class: typing.Type[typing.Any], buffer: bytes, *args) -> typing.Any:
"""
Deserializes byte buffer to the generated object.
This method can potentially use all bits of the last byte even if not all of them were written during
serialization (because there is no way how to specify exact number of bits). Thus, it could allow reading
behind stream (possibly in case of damaged data).
:param obj_class: Class instance of the generated object to deserialize.
:param buffer: Byte buffer which represents generated object in binary format.
:param args: Additional arguments needed for obj_class.from_reader method.
:returns: Generated object created from given byte buffer.
:raises PythonRuntimeException: Throws in case of any error during deserialization.
"""
bitbuffer = BitBuffer(buffer)
return deserialize(obj_class, bitbuffer, *args)
def test_hashcode(self):
bitsize = 11
bitbuffer1 = BitBuffer(bytes([0xAB, 0xE0]), bitsize)
bitbuffer2 = BitBuffer(bytes([0xAB, 0xF0]), bitsize)
self.assertEqual(hash(bitbuffer1), hash(bitbuffer2))
bitbuffer3 = BitBuffer(bytes([0xAB, 0xFF]), bitsize)
self.assertEqual(hash(bitbuffer1), hash(bitbuffer3))
bitbuffer4 = BitBuffer(bytes([0xAB, 0xC0]), bitsize)
self.assertNotEqual(hash(bitbuffer1), hash(bitbuffer4))
bitbuffer5 = BitBuffer(bytes([0xBA, 0xE0]), bitsize)
self.assertNotEqual(hash(bitbuffer1), hash(bitbuffer5))
bitbuffer6 = BitBuffer(bytes([0xAB]))
self.assertNotEqual(hash(bitbuffer1), hash(bitbuffer6))
bitbuffer7 = BitBuffer(bytes())
self.assertNotEqual(hash(bitbuffer1), hash(bitbuffer7))
def testHashCode(self):
bitSize = 11
bitBuffer1 = BitBuffer(bytes([0xAB, 0x07]), bitSize)
bitBuffer2 = BitBuffer(bytes([0xAB, 0x0F]), bitSize)
self.assertEqual(hash(bitBuffer1), hash(bitBuffer2))
bitBuffer3 = BitBuffer(bytes([0xAB, 0xFF]), bitSize)
self.assertEqual(hash(bitBuffer1), hash(bitBuffer3))
bitBuffer4 = BitBuffer(bytes([0xAB, 0x03]), bitSize)
self.assertNotEqual(hash(bitBuffer1), hash(bitBuffer4))
bitBuffer5 = BitBuffer(bytes([0xBA, 0x07]), bitSize)
self.assertNotEqual(hash(bitBuffer1), hash(bitBuffer5))
bitBuffer6 = BitBuffer(bytes([0xAB]))
self.assertNotEqual(hash(bitBuffer1), hash(bitBuffer6))
bitBuffer7 = BitBuffer(bytes())
self.assertNotEqual(hash(bitBuffer1), hash(bitBuffer7))
def testGetBitSizeOfBitBuffer(self):
testBitBuffer1 = BitBuffer([0xAB, 0x03], 8)
self.assertEqual(8 + 8, getBitSizeOfBitBuffer(testBitBuffer1))
testBitBuffer2 = BitBuffer([0xAB, 0x03], 11)
self.assertEqual(8 + 11, getBitSizeOfBitBuffer(testBitBuffer2))
testBitBuffer3 = BitBuffer([0xAB, 0xCD], 16)
self.assertEqual(8 + 16, getBitSizeOfBitBuffer(testBitBuffer3))
testBitBuffer4 = BitBuffer([0xAB, 0xCD])
self.assertEqual(8 + 16, getBitSizeOfBitBuffer(testBitBuffer4))
testBitBuffer5 = BitBuffer(16 * [1], 127)
self.assertEqual(8 + 15 * 8 + 7, getBitSizeOfBitBuffer(testBitBuffer5))
testBitBuffer6 = BitBuffer(16 * [1], 128)
self.assertEqual(16 + 16 * 8, getBitSizeOfBitBuffer(testBitBuffer6))
def test_bitsizeof_bitbuffer(self):
test_bitbuffer1 = BitBuffer([0xAB, 0x03], 8)
self.assertEqual(8 + 8, bitsizeof_bitbuffer(test_bitbuffer1))
test_bitbuffer2 = BitBuffer([0xAB, 0x03], 11)
self.assertEqual(8 + 11, bitsizeof_bitbuffer(test_bitbuffer2))
test_bitbuffer3 = BitBuffer([0xAB, 0xCD], 16)
self.assertEqual(8 + 16, bitsizeof_bitbuffer(test_bitbuffer3))
test_bitbuffer4 = BitBuffer([0xAB, 0xCD])
self.assertEqual(8 + 16, bitsizeof_bitbuffer(test_bitbuffer4))
test_bitbuffer5 = BitBuffer(16 * [1], 127)
self.assertEqual(8 + 15 * 8 + 7, bitsizeof_bitbuffer(test_bitbuffer5))
test_bitbuffer6 = BitBuffer(16 * [1], 128)
self.assertEqual(16 + 16 * 8, bitsizeof_bitbuffer(test_bitbuffer6))
def test_eq(self):
bitsize = 11
bitbuffer1 = BitBuffer(bytes([0xAB, 0xE0]), bitsize)
bitbuffer2 = BitBuffer(bytes([0xAB, 0xF0]), bitsize)
self.assertEqual(bitbuffer1, bitbuffer2)
bitbuffer3 = BitBuffer(bytes([0xAB, 0xFF]), bitsize)
self.assertEqual(bitbuffer1, bitbuffer3)
bitbuffer4 = BitBuffer(bytes([0xAB, 0xC0]), bitsize)
self.assertNotEqual(bitbuffer1, bitbuffer4)
bitbuffer5 = BitBuffer(bytes([0xBA, 0xE0]), bitsize)
self.assertNotEqual(bitbuffer1, bitbuffer5)
bitbuffer6 = BitBuffer(bytes([0xAB]))
self.assertNotEqual(bitbuffer1, bitbuffer6)
bitbuffer7 = BitBuffer(bytes())
self.assertNotEqual(bitbuffer1, bitbuffer7)
self.assertNotEqual(bitbuffer1, 1)
def testEq(self):
bitSize = 11
bitBuffer1 = BitBuffer(bytes([0xAB, 0x07]), bitSize)
bitBuffer2 = BitBuffer(bytes([0xAB, 0x0F]), bitSize)
self.assertEqual(bitBuffer1, bitBuffer2)
bitBuffer3 = BitBuffer(bytes([0xAB, 0xFF]), bitSize)
self.assertEqual(bitBuffer1, bitBuffer3)
bitBuffer4 = BitBuffer(bytes([0xAB, 0x03]), bitSize)
self.assertNotEqual(bitBuffer1, bitBuffer4)
bitBuffer5 = BitBuffer(bytes([0xBA, 0x07]), bitSize)
self.assertNotEqual(bitBuffer1, bitBuffer5)
bitBuffer6 = BitBuffer(bytes([0xAB]))
self.assertNotEqual(bitBuffer1, bitBuffer6)
bitBuffer7 = BitBuffer(bytes())
self.assertNotEqual(bitBuffer1, bitBuffer7)
self.assertNotEqual(bitBuffer1, 1)
def testGetBitSize(self):
bitSize = 11
bitBuffer = BitBuffer(bytes([0xAB, 0x07]), bitSize)
self.assertEqual(bitSize, bitBuffer.getBitSize())
def test_buffer(self):
bitsize = 11
buffer = bytes([0xAB, 0xE0])
bitbuffer = BitBuffer(buffer, bitsize)
self.assertEqual(buffer, bitbuffer.buffer)
def testGetByteSize(self):
bitSize = 11
buffer = bytes([0xAB, 0x07])
byteSize = len(buffer)
bitBuffer = BitBuffer(buffer, bitSize)
self.assertEqual(byteSize, bitBuffer.getByteSize())
def test_bitsize(self):
bitsize = 11
bitbuffer = BitBuffer(bytes([0xAB, 0xE0]), bitsize)
self.assertEqual(bitsize, bitbuffer.bitsize)