def test_create_wrap_bytearray(self):
ba = bytearray(b'\x03\x05\x07\x09')
val = ByteBuffer(ba)
assert val.position == 0
assert val.capacity == 4
assert val.remaining == 4
assert val.get(4) == ba
def test_create_wrap_bytearray(self):
ba = bytearray(b'\x03\x05\x07\x09')
val = ByteBuffer(ba)
assert val.position == 0
assert val.capacity == 4
assert val.remaining == 4
assert val.get(4) == ba
def test_decode_sequence(self):
schema = [{'name': 'bar', 'type': 'int8'}]
assert _decode_sequence(
schema,
ByteBuffer(
bytearray(b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00'))) == {
'bar': 1
}
schema = [{'name': 'bar', 'type': 'array', 'item_type': 'int8'}]
assert _decode_sequence(
schema,
ByteBuffer(
bytearray(b'\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00'))) == {
'bar': [1]
}
def test_encode_plain_value_errors(self):
buf = ByteBuffer(10)
self.assertRaises(NotImplementedError, _encode_plain_value, {'foo': 1},
{
'name': 'foo',
'type': 'unknowntype'
}, buf)
def test_decode_array(self):
assert _decode_array(
'int8',
ByteBuffer(
bytearray(b'\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00'))) == [
1
]
assert _decode_array(
'int8',
ByteBuffer(bytearray(
b'\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00'))) is None
assert _decode_array([{
'name': 'bar',
'type': 'int8'
}], ByteBuffer(
bytearray(b'\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00'))) == [{
'bar':
1
}]
def set_assignment(self):
assignment = MemberAssignmentV0.from_bytebuffer(
0, ByteBuffer(self.assignment_data))
self.assignment_version = assignment['version']
self.user_data = assignment['user_data']
for tp in assignment['partitions']:
if tp['topic'] not in self.topics:
self.topics[tp['topic']] = []
for partition in tp['partitions']:
self.topics[tp['topic']].append(partition)
def test_encode_sequence(self):
buf = ByteBuffer(10)
_encode_sequence({'foo': 1}, [{'name': 'foo', 'type': 'int8'}], buf)
assert buf.get(10, 0) == bytearray(b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_sequence({'foo': None}, [{'name': 'foo', 'type': 'array', 'item_type': 'int8'}], buf)
assert buf.get(10, 0) == bytearray(b'\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_sequence({'foo': [1]}, [{'name': 'foo', 'type': 'array', 'item_type': 'int8'}], buf)
assert buf.get(10, 0) == bytearray(b'\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_sequence({'foo': [{'bar': 1}]}, [{'name': 'foo', 'type': 'array', 'item_type': [{'name': 'bar', 'type': 'int8'}]}], buf)
assert buf.get(10, 0) == bytearray(b'\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00')
def test_decode_plain_value(self):
assert _decode_plain_type(
'int8',
ByteBuffer(
bytearray(b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00'))) == 1
assert _decode_plain_type(
'int16',
ByteBuffer(
bytearray(b'\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00'))) == 1
assert _decode_plain_type(
'int32',
ByteBuffer(
bytearray(b'\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00'))) == 1
assert _decode_plain_type(
'int64',
ByteBuffer(
bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00'))) == 1
assert _decode_plain_type(
'boolean',
ByteBuffer(bytearray(
b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00'))) is True
assert _decode_plain_type(
'string',
ByteBuffer(bytearray(b'\x00\x03foo\x00\x00\x00\x00\x00'))) == 'foo'
assert _decode_plain_type(
'string',
ByteBuffer(bytearray(
b'\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00'))) is None
assert _decode_plain_type(
'bytes',
ByteBuffer(bytearray(
b'\x00\x00\x00\x02\x78\x23\x00\x00\x00\x00'))) == b'\x78\x23'
assert _decode_plain_type(
'bytes',
ByteBuffer(bytearray(
b'\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00'))) is None
def _single_send(self, request):
# Build the payload based on the request passed in. We'll fill in the size at the end
buf = ByteBuffer(self._configuration.max_request_size)
buf.putInt32(0)
buf.putInt16(request.api_key)
buf.putInt16(request.api_version)
buf.putInt32(self._correlation_id)
buf.putInt16(len(self._configuration.client_id))
buf.put(struct.pack('{0}s'.format(len(self._configuration.client_id)), self._configuration.client_id.encode("utf-8")))
request.encode(buf)
# Close the payload and write the size (payload size without the size field itself)
buf.limit = buf.position - 1
payload_len = buf.capacity - 4
buf.rewind()
buf.putInt32(payload_len)
buf.rewind()
# Increment the correlation ID for the next request
self._correlation_id += 1
try:
# Send the payload bytes to the broker
self._sock.sendall(buf.get(buf.capacity))
# Read the first 4 bytes so we know the size
size = ByteBuffer(self._sock.recv(4)).getInt32()
# Read the response that we're expecting
response_data = self._read_bytes(size)
response = ByteBuffer(response_data)
# Parse off the correlation ID for the response
correlation_id = response.getInt32()
except EOFError:
raise ConnectionError("Failed to read enough data from Kafka")
except socket.error as e:
raise ConnectionError("Failed communicating with Kafka: {0}".format(e))
# Get the proper response class and parse the response
return correlation_id, request.response.from_bytebuffer(correlation_id, response.slice())
def setUp(self):
self.test_bb = ByteBuffer(b'\x03\x05\x07\x09\x11\x13\x15\x17\x19\x21')
class ByteBufferTests(unittest.TestCase):
def setUp(self):
self.test_bb = ByteBuffer(b'\x03\x05\x07\x09\x11\x13\x15\x17\x19\x21')
def test_create_allocate(self):
val = ByteBuffer(4)
assert val.position == 0
assert val.capacity == 4
assert val.remaining == 4
def test_create_wrap_bytestring(self):
val = ByteBuffer(b'\x03\x05\x07\x09')
assert val.position == 0
assert val.capacity == 4
assert val.remaining == 4
assert val.get(4) == bytearray(b'\x03\x05\x07\x09')
def test_create_wrap_bytearray(self):
ba = bytearray(b'\x03\x05\x07\x09')
val = ByteBuffer(ba)
assert val.position == 0
assert val.capacity == 4
assert val.remaining == 4
assert val.get(4) == ba
def test_create_bad_type(self):
self.assertRaises(TypeError, ByteBuffer, [])
def test_get_int8(self):
assert self.test_bb.getInt8() == 3
assert self.test_bb.position == 1
def test_get_int8_at_position(self):
assert self.test_bb.getInt8(2) == 7
assert self.test_bb.position == 0
def test_get_int16(self):
assert self.test_bb.getInt16() == 773
assert self.test_bb.position == 2
def test_get_int16_at_position(self):
assert self.test_bb.getInt16(2) == 1801
assert self.test_bb.position == 0
def test_get_int32(self):
assert self.test_bb.getInt32() == 50661129
assert self.test_bb.position == 4
def test_get_int32_at_position(self):
assert self.test_bb.getInt32(2) == 118034707
assert self.test_bb.position == 0
def test_get_int64(self):
assert self.test_bb.getInt64() == 217587892519900439
assert self.test_bb.position == 8
def test_get_int64_at_position(self):
assert self.test_bb.getInt64(2) == 506955206711777569
assert self.test_bb.position == 0
def test_get(self):
assert self.test_bb.get(4) == bytearray(b'\x03\x05\x07\x09')
def test_get_at_position(self):
assert self.test_bb.get(4, 2) == bytearray(b'\x07\x09\x11\x13')
def test_put_int8(self):
self.test_bb.putInt8(8)
assert self.test_bb.get(10, 0) == bytearray(b'\x08\x05\x07\x09\x11\x13\x15\x17\x19\x21')
def test_put_int8_at_position(self):
self.test_bb.putInt8(8, 2)
assert self.test_bb.get(10, 0) == bytearray(b'\x03\x05\x08\x09\x11\x13\x15\x17\x19\x21')
def test_put_int16(self):
self.test_bb.putInt16(2342)
assert self.test_bb.get(10, 0) == bytearray(b'\t&\x07\x09\x11\x13\x15\x17\x19\x21')
def test_put_int16_at_position(self):
self.test_bb.putInt16(2342, 2)
assert self.test_bb.get(10, 0) == bytearray(b'\x03\x05\t&\x11\x13\x15\x17\x19\x21')
def test_put_int32(self):
self.test_bb.putInt32(897324)
assert self.test_bb.get(10, 0) == bytearray(b'\x00\r\xb1,\x11\x13\x15\x17\x19\x21')
def test_put_int32_at_position(self):
self.test_bb.putInt32(897324, 2)
assert self.test_bb.get(10, 0) == bytearray(b'\x03\x05\x00\r\xb1,\x15\x17\x19\x21')
def test_put_int64(self):
self.test_bb.putInt64(7992478585871677467)
assert self.test_bb.get(10, 0) == bytearray(b'n\xea\xfc\xed\x89\xcc8\x1b\x19\x21')
def test_put_int64_at_position(self):
self.test_bb.putInt64(7992478585871677467, 2)
assert self.test_bb.get(10, 0) == bytearray(b'\x03\x05n\xea\xfc\xed\x89\xcc8\x1b')
def test_put(self):
self.test_bb.put(b'\x21\x23\x25\x27')
assert self.test_bb.get(10, 0) == bytearray(b'\x21\x23\x25\x27\x11\x13\x15\x17\x19\x21')
def test_put_at_position(self):
self.test_bb.put(b'\x21\x23\x25\x27', 2)
assert self.test_bb.get(10, 0) == bytearray(b'\x03\x05\x21\x23\x25\x27\x15\x17\x19\x21')
def test_put_bad_type(self):
self.assertRaises(TypeError, self.test_bb.put, 56)
def test_underflow(self):
self.assertRaises(EOFError, self.test_bb.getInt64, 3)
def test_position_setter(self):
self.test_bb.position = 3
assert self.test_bb.position == 3
assert self.test_bb.getInt8() == 9
def test_position_setter_out_of_range(self):
with self.assertRaises(IndexError):
self.test_bb.position = 10
def test_position_setter_bad_type(self):
with self.assertRaises(TypeError):
self.test_bb.position = 'nonint'
def test_position_setter_out_of_range_negative(self):
with self.assertRaises(IndexError):
self.test_bb.position = -1
def test_capacity(self):
assert self.test_bb.capacity == 10
def test_limit(self):
assert self.test_bb.limit == 9
def test_limit_setter(self):
self.test_bb.limit = 4
self.assertRaises(EOFError, self.test_bb.getInt64)
def test_limit_setter_out_of_range(self):
with self.assertRaises(IndexError):
self.test_bb.limit = 10
def test_limit_setter_bad_type(self):
with self.assertRaises(TypeError):
self.test_bb.limit = 'nonint'
def test_rewind(self):
self.test_bb.position = 4
self.test_bb.rewind()
assert self.test_bb.position == 0
def test_duplicate(self):
val = self.test_bb.duplicate()
assert val._buffer is self.test_bb._buffer
assert val._first == self.test_bb._first
assert val.limit == self.test_bb.limit
assert val.position == self.test_bb.position
def test_slice(self):
self.test_bb.position = 4
val = self.test_bb.slice()
assert val._buffer is self.test_bb._buffer
assert val._first == 4
assert val.limit == self.test_bb.limit
assert val.position == 4
def test_slice_rewind(self):
self.test_bb.position = 4
val = self.test_bb.slice()
val.rewind()
assert val.position == 4
def test_encode_plain_value(self):
buf = ByteBuffer(10)
_encode_plain_value(1, 'int8', buf)
assert buf.get(
10, 0) == bytearray(b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(1, 'int16', buf)
assert buf.get(
10, 0) == bytearray(b'\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(1, 'int32', buf)
assert buf.get(
10, 0) == bytearray(b'\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(1, 'int64', buf)
assert buf.get(
10, 0) == bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(True, 'boolean', buf)
assert buf.get(
10, 0) == bytearray(b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value('foo', 'string', buf)
assert buf.get(10, 0) == bytearray(b'\x00\x03foo\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(None, 'string', buf)
assert buf.get(
10, 0) == bytearray(b'\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(b'\x78\x23', 'bytes', buf)
assert buf.get(
10, 0) == bytearray(b'\x00\x00\x00\x02\x78\x23\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(None, 'bytes', buf)
assert buf.get(
10, 0) == bytearray(b'\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00')
class ByteBufferTests(unittest.TestCase):
def setUp(self):
self.test_bb = ByteBuffer(b'\x03\x05\x07\x09\x11\x13\x15\x17\x19\x21')
def test_create_allocate(self):
val = ByteBuffer(4)
assert val.position == 0
assert val.capacity == 4
assert val.remaining == 4
def test_create_wrap_bytestring(self):
val = ByteBuffer(b'\x03\x05\x07\x09')
assert val.position == 0
assert val.capacity == 4
assert val.remaining == 4
assert val.get(4) == bytearray(b'\x03\x05\x07\x09')
def test_create_wrap_bytearray(self):
ba = bytearray(b'\x03\x05\x07\x09')
val = ByteBuffer(ba)
assert val.position == 0
assert val.capacity == 4
assert val.remaining == 4
assert val.get(4) == ba
def test_create_bad_type(self):
self.assertRaises(TypeError, ByteBuffer, [])
def test_get_int8(self):
assert self.test_bb.getInt8() == 3
assert self.test_bb.position == 1
def test_get_int8_at_position(self):
assert self.test_bb.getInt8(2) == 7
assert self.test_bb.position == 0
def test_get_int16(self):
assert self.test_bb.getInt16() == 773
assert self.test_bb.position == 2
def test_get_int16_at_position(self):
assert self.test_bb.getInt16(2) == 1801
assert self.test_bb.position == 0
def test_get_int32(self):
assert self.test_bb.getInt32() == 50661129
assert self.test_bb.position == 4
def test_get_int32_at_position(self):
assert self.test_bb.getInt32(2) == 118034707
assert self.test_bb.position == 0
def test_get_int64(self):
assert self.test_bb.getInt64() == 217587892519900439
assert self.test_bb.position == 8
def test_get_int64_at_position(self):
assert self.test_bb.getInt64(2) == 506955206711777569
assert self.test_bb.position == 0
def test_get(self):
assert self.test_bb.get(4) == bytearray(b'\x03\x05\x07\x09')
def test_get_at_position(self):
assert self.test_bb.get(4, 2) == bytearray(b'\x07\x09\x11\x13')
def test_put_int8(self):
self.test_bb.putInt8(8)
assert self.test_bb.get(
10, 0) == bytearray(b'\x08\x05\x07\x09\x11\x13\x15\x17\x19\x21')
def test_put_int8_at_position(self):
self.test_bb.putInt8(8, 2)
assert self.test_bb.get(
10, 0) == bytearray(b'\x03\x05\x08\x09\x11\x13\x15\x17\x19\x21')
def test_put_int16(self):
self.test_bb.putInt16(2342)
assert self.test_bb.get(
10, 0) == bytearray(b'\t&\x07\x09\x11\x13\x15\x17\x19\x21')
def test_put_int16_at_position(self):
self.test_bb.putInt16(2342, 2)
assert self.test_bb.get(
10, 0) == bytearray(b'\x03\x05\t&\x11\x13\x15\x17\x19\x21')
def test_put_int32(self):
self.test_bb.putInt32(897324)
assert self.test_bb.get(
10, 0) == bytearray(b'\x00\r\xb1,\x11\x13\x15\x17\x19\x21')
def test_put_int32_at_position(self):
self.test_bb.putInt32(897324, 2)
assert self.test_bb.get(
10, 0) == bytearray(b'\x03\x05\x00\r\xb1,\x15\x17\x19\x21')
def test_put_int64(self):
self.test_bb.putInt64(7992478585871677467)
assert self.test_bb.get(
10, 0) == bytearray(b'n\xea\xfc\xed\x89\xcc8\x1b\x19\x21')
def test_put_int64_at_position(self):
self.test_bb.putInt64(7992478585871677467, 2)
assert self.test_bb.get(
10, 0) == bytearray(b'\x03\x05n\xea\xfc\xed\x89\xcc8\x1b')
def test_put(self):
self.test_bb.put(b'\x21\x23\x25\x27')
assert self.test_bb.get(
10, 0) == bytearray(b'\x21\x23\x25\x27\x11\x13\x15\x17\x19\x21')
def test_put_at_position(self):
self.test_bb.put(b'\x21\x23\x25\x27', 2)
assert self.test_bb.get(
10, 0) == bytearray(b'\x03\x05\x21\x23\x25\x27\x15\x17\x19\x21')
def test_put_bad_type(self):
self.assertRaises(TypeError, self.test_bb.put, 56)
def test_underflow(self):
self.assertRaises(EOFError, self.test_bb.getInt64, 3)
def test_position_setter(self):
self.test_bb.position = 3
assert self.test_bb.position == 3
assert self.test_bb.getInt8() == 9
def test_position_setter_out_of_range(self):
with self.assertRaises(IndexError):
self.test_bb.position = 10
def test_position_setter_bad_type(self):
with self.assertRaises(TypeError):
self.test_bb.position = 'nonint'
def test_position_setter_out_of_range_negative(self):
with self.assertRaises(IndexError):
self.test_bb.position = -1
def test_capacity(self):
assert self.test_bb.capacity == 10
def test_limit(self):
assert self.test_bb.limit == 9
def test_limit_setter(self):
self.test_bb.limit = 4
self.assertRaises(EOFError, self.test_bb.getInt64)
def test_limit_setter_out_of_range(self):
with self.assertRaises(IndexError):
self.test_bb.limit = 10
def test_limit_setter_bad_type(self):
with self.assertRaises(TypeError):
self.test_bb.limit = 'nonint'
def test_rewind(self):
self.test_bb.position = 4
self.test_bb.rewind()
assert self.test_bb.position == 0
def test_duplicate(self):
val = self.test_bb.duplicate()
assert val._buffer is self.test_bb._buffer
assert val._first == self.test_bb._first
assert val.limit == self.test_bb.limit
assert val.position == self.test_bb.position
def test_slice(self):
self.test_bb.position = 4
val = self.test_bb.slice()
assert val._buffer is self.test_bb._buffer
assert val._first == 4
assert val.limit == self.test_bb.limit
assert val.position == 4
def test_slice_rewind(self):
self.test_bb.position = 4
val = self.test_bb.slice()
val.rewind()
assert val.position == 4
def setUp(self):
self.test_bb = ByteBuffer(b'\x03\x05\x07\x09\x11\x13\x15\x17\x19\x21')
def test_decode_plain_value_errors(self):
self.assertRaises(NotImplementedError, _decode_plain_type,
'unknowntype', ByteBuffer(10))
def test_encode_sequence(self):
buf = ByteBuffer(10)
_encode_sequence({'foo': 1}, [{'name': 'foo', 'type': 'int8'}], buf)
assert buf.get(
10, 0) == bytearray(b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_sequence({'foo': None}, [{
'name': 'foo',
'type': 'array',
'item_type': 'int8'
}], buf)
assert buf.get(
10, 0) == bytearray(b'\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_sequence({'foo': [1]}, [{
'name': 'foo',
'type': 'array',
'item_type': 'int8'
}], buf)
assert buf.get(
10, 0) == bytearray(b'\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_sequence({'foo': [{
'bar': 1
}]}, [{
'name': 'foo',
'type': 'array',
'item_type': [{
'name': 'bar',
'type': 'int8'
}]
}], buf)
assert buf.get(
10, 0) == bytearray(b'\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00')
def test_encode_plain_value(self):
buf = ByteBuffer(10)
_encode_plain_value(1, 'int8', buf)
assert buf.get(10, 0) == bytearray(b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(1, 'int16', buf)
assert buf.get(10, 0) == bytearray(b'\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(1, 'int32', buf)
assert buf.get(10, 0) == bytearray(b'\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(1, 'int64', buf)
assert buf.get(10, 0) == bytearray(b'\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(True, 'boolean', buf)
assert buf.get(10, 0) == bytearray(b'\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value('foo', 'string', buf)
assert buf.get(10, 0) == bytearray(b'\x00\x03foo\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(None, 'string', buf)
assert buf.get(10, 0) == bytearray(b'\xff\xff\x00\x00\x00\x00\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(b'\x78\x23', 'bytes', buf)
assert buf.get(10, 0) == bytearray(b'\x00\x00\x00\x02\x78\x23\x00\x00\x00\x00')
buf = ByteBuffer(10)
_encode_plain_value(None, 'bytes', buf)
assert buf.get(10, 0) == bytearray(b'\xff\xff\xff\xff\x00\x00\x00\x00\x00\x00')
def test_create_allocate(self):
val = ByteBuffer(4)
assert val.position == 0
assert val.capacity == 4
assert val.remaining == 4