def test_decode_fetch_response(self): t1 = "topic1" t2 = u"topic2" msgs = [create_message(m) for m in [b"message1", b"hi", b"boo", b"foo", b"so fun!"]] ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]]) ms2 = KafkaCodec._encode_message_set([msgs[2]]) ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]]) encoded = struct.pack('>iih%dsiihqi%dsihqi%dsh%dsiihqi%ds' % (len(t1), len(ms1), len(ms2), len(t2), len(ms3)), 4, 2, len(t1), t1.encode(), 2, 0, 0, 10, len(ms1), ms1, 1, 1, 20, len(ms2), ms2, len(t2), t2.encode(), 1, 0, 0, 30, len(ms3), ms3) responses = list(KafkaCodec.decode_fetch_response(encoded)) def expand_messages(response): return FetchResponse(response.topic, response.partition, response.error, response.highwaterMark, list(response.messages)) expanded_responses = [expand_messages(r) for r in responses] expect = [FetchResponse(t1, 0, 0, 10, [OffsetAndMessage(0, msgs[0]), OffsetAndMessage(0, msgs[1])]), FetchResponse(t1, 1, 1, 20, [OffsetAndMessage(0, msgs[2])]), FetchResponse(t2, 0, 0, 30, [OffsetAndMessage(0, msgs[3]), OffsetAndMessage(0, msgs[4])])] self.assertEqual(expanded_responses, expect)
def test_decode_fetch_response(self): t1 = "topic1" t2 = "topic2" msgs = map(create_message, ["message1", "hi", "boo", "foo", "so fun!"]) ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]]) ms2 = KafkaCodec._encode_message_set([msgs[2]]) ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]]) encoded = struct.pack( '>iih%dsiihqi%dsihqi%dsh%dsiihqi%ds' % (len(t1), len(ms1), len(ms2), len(t2), len(ms3)), 4, 2, len(t1), t1, 2, 0, 0, 10, len(ms1), ms1, 1, 1, 20, len(ms2), ms2, len(t2), t2, 1, 0, 0, 30, len(ms3), ms3) responses = list(KafkaCodec.decode_fetch_response(encoded)) def expand_messages(response): return FetchResponse(response.topic, response.partition, response.error, response.highwaterMark, list(response.messages)) expanded_responses = map(expand_messages, responses) expect = [ FetchResponse( t1, 0, 0, 10, [OffsetAndMessage(0, msgs[0]), OffsetAndMessage(0, msgs[1])]), FetchResponse(t1, 1, 1, 20, [OffsetAndMessage(0, msgs[2])]), FetchResponse( t2, 0, 0, 30, [OffsetAndMessage(0, msgs[3]), OffsetAndMessage(0, msgs[4])]) ] self.assertEqual(expanded_responses, expect)
def make_fetch_response(id): t1 = "topic1" t2 = "topic2" msgs = map( create_message, ["message1", "hi", "boo", "foo", "so fun!"]) ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]]) ms2 = KafkaCodec._encode_message_set([msgs[2]]) ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]]) packFmt = '>iih{}siihqi{}sihqi{}sh{}siihqi{}s'.format( len(t1), len(ms1), len(ms2), len(t2), len(ms3)) return struct.pack( packFmt, id, 2, len(t1), t1, 2, 0, 0, 10, len(ms1), ms1, 1, 1, 20, len(ms2), ms2, len(t2), t2, 1, 0, 0, 30, len(ms3), ms3)
def make_fetch_response(id): t1 = "topic1" t2 = "topic2" msgs = map(create_message, ["message1", "hi", "boo", "foo", "so fun!"]) ms1 = KafkaCodec._encode_message_set([msgs[0], msgs[1]]) ms2 = KafkaCodec._encode_message_set([msgs[2]]) ms3 = KafkaCodec._encode_message_set([msgs[3], msgs[4]]) packFmt = '>iih{}siihqi{}sihqi{}sh{}siihqi{}s'.format( len(t1), len(ms1), len(ms2), len(t2), len(ms3)) return struct.pack(packFmt, id, 2, len(t1), t1, 2, 0, 0, 10, len(ms1), ms1, 1, 1, 20, len(ms2), ms2, len(t2), t2, 1, 0, 0, 30, len(ms3), ms3)
def test_encode_message_set(self): message_set = [ create_message(b"v1", b"k1"), create_message(b"v2", b"k2"), ] encoded = KafkaCodec._encode_message_set(message_set) expect = b"".join([ struct.pack(">q", 0), # MsgSet Offset struct.pack(">i", 18), # Msg Size struct.pack(">i", 1474775406), # CRC struct.pack(">bb", 0, 0), # Magic, flags struct.pack(">i", 2), # Length of key b"k1", # Key struct.pack(">i", 2), # Length of value b"v1", # Value struct.pack(">q", 0), # MsgSet Offset struct.pack(">i", 18), # Msg Size struct.pack(">i", -16383415), # CRC struct.pack(">bb", 0, 0), # Magic, flags struct.pack(">i", 2), # Length of key b"k2", # Key struct.pack(">i", 2), # Length of value b"v2", # Value ]) self.assertEqual(encoded, expect)
def test_encode_message_set(self): message_set = [ create_message(b"v1", b"k1"), create_message(b"v2", b"k2") ] encoded = KafkaCodec._encode_message_set(message_set) expect = b"".join([ struct.pack(">q", 0), # MsgSet Offset struct.pack(">i", 18), # Msg Size struct.pack(">i", 1474775406), # CRC struct.pack(">bb", 0, 0), # Magic, flags struct.pack(">i", 2), # Length of key b"k1", # Key struct.pack(">i", 2), # Length of value b"v1", # Value struct.pack(">q", 0), # MsgSet Offset struct.pack(">i", 18), # Msg Size struct.pack(">i", -16383415), # CRC struct.pack(">bb", 0, 0), # Magic, flags struct.pack(">i", 2), # Length of key b"k2", # Key struct.pack(">i", 2), # Length of value b"v2", # Value ]) self.assertEqual(encoded, expect)
def test_encode_message_set(self): message_set = [ create_message((-1, b"v1"), b"k1"), create_message((-1, b"v2"), b"k2"), ] encoded = KafkaCodec._encode_message_set(message_set) expect = b"".join([ struct.pack(">q", 0), # MsgSet Offset struct.pack(">i", 26), # Msg Size struct.pack(">i", -634178223), # CRC struct.pack(">bbq", 1, 0, -1), # Magic, flags, timestamp struct.pack(">i", 2), # Length of key b"k1", # Key struct.pack(">i", 2), # Length of value b"v1", # Value struct.pack(">q", 0), # MsgSet Offset struct.pack(">i", 26), # Msg Size struct.pack(">i", 1926397558), # CRC struct.pack(">bbq", 1, 0, -1), # Magic, flags, timestamp struct.pack(">i", 2), # Length of key b"k2", # Key struct.pack(">i", 2), # Length of value b"v2", # Value ]) self.assertEqual(encoded, expect)