示例#1
0
    def test_encode_produce_request(self):
        requests = [
            ProduceRequestPayload("topic1", 0, [
                kafka.protocol.message.Message(b"a"),
                kafka.protocol.message.Message(b"b")
            ]),
            ProduceRequestPayload("topic2", 1,
                                  [kafka.protocol.message.Message(b"c")])
        ]

        msg_a_binary = KafkaProtocol._encode_message(create_message(b"a"))
        msg_b_binary = KafkaProtocol._encode_message(create_message(b"b"))
        msg_c_binary = KafkaProtocol._encode_message(create_message(b"c"))

        header = b"".join([
            struct.pack('>i', 0x94),  # The length of the message overall
            struct.pack('>h', 0),  # Msg Header, Message type = Produce
            struct.pack('>h', 0),  # Msg Header, API version
            struct.pack('>i', 2),  # Msg Header, Correlation ID
            struct.pack('>h7s', 7, b"client1"),  # Msg Header, The client ID
            struct.pack('>h', 2),  # Num acks required
            struct.pack('>i', 100),  # Request Timeout
            struct.pack('>i', 2),  # The number of requests
        ])

        total_len = len(msg_a_binary) + len(msg_b_binary)
        topic1 = b"".join([
            struct.pack('>h6s', 6, b'topic1'),  # The topic1
            struct.pack('>i', 1),  # One message set
            struct.pack('>i', 0),  # Partition 0
            struct.pack('>i',
                        total_len + 24),  # Size of the incoming message set
            struct.pack('>q', 0),  # No offset specified
            struct.pack('>i', len(msg_a_binary)),  # Length of message
            msg_a_binary,  # Actual message
            struct.pack('>q', 0),  # No offset specified
            struct.pack('>i', len(msg_b_binary)),  # Length of message
            msg_b_binary,  # Actual message
        ])

        topic2 = b"".join([
            struct.pack('>h6s', 6, b'topic2'),  # The topic1
            struct.pack('>i', 1),  # One message set
            struct.pack('>i', 1),  # Partition 1
            struct.pack('>i',
                        len(msg_c_binary) +
                        12),  # Size of the incoming message set
            struct.pack('>q', 0),  # No offset specified
            struct.pack('>i', len(msg_c_binary)),  # Length of message
            msg_c_binary,  # Actual message
        ])

        expected1 = b"".join([header, topic1, topic2])
        expected2 = b"".join([header, topic2, topic1])

        encoded = KafkaProtocol.encode_produce_request(b"client1", 2, requests,
                                                       2, 100)
        self.assertIn(encoded, [expected1, expected2])
    def test_encode_produce_request(self):
        requests = [
            ProduceRequestPayload("topic1", 0, [
                kafka.protocol.message.Message(b"a"),
                kafka.protocol.message.Message(b"b")
            ]),
            ProduceRequestPayload("topic2", 1, [
                kafka.protocol.message.Message(b"c")
            ])
        ]

        msg_a_binary = KafkaProtocol._encode_message(create_message(b"a"))
        msg_b_binary = KafkaProtocol._encode_message(create_message(b"b"))
        msg_c_binary = KafkaProtocol._encode_message(create_message(b"c"))

        header = b"".join([
            struct.pack('>i', 0x94),                   # The length of the message overall
            struct.pack('>h', 0),                      # Msg Header, Message type = Produce
            struct.pack('>h', 0),                      # Msg Header, API version
            struct.pack('>i', 2),                      # Msg Header, Correlation ID
            struct.pack('>h7s', 7, b"client1"),        # Msg Header, The client ID
            struct.pack('>h', 2),                      # Num acks required
            struct.pack('>i', 100),                    # Request Timeout
            struct.pack('>i', 2),                      # The number of requests
        ])

        total_len = len(msg_a_binary) + len(msg_b_binary)
        topic1 = b"".join([
            struct.pack('>h6s', 6, b'topic1'),         # The topic1
            struct.pack('>i', 1),                      # One message set
            struct.pack('>i', 0),                      # Partition 0
            struct.pack('>i', total_len + 24),         # Size of the incoming message set
            struct.pack('>q', 0),                      # No offset specified
            struct.pack('>i', len(msg_a_binary)),      # Length of message
            msg_a_binary,                              # Actual message
            struct.pack('>q', 0),                      # No offset specified
            struct.pack('>i', len(msg_b_binary)),      # Length of message
            msg_b_binary,                              # Actual message
        ])

        topic2 = b"".join([
            struct.pack('>h6s', 6, b'topic2'),         # The topic1
            struct.pack('>i', 1),                      # One message set
            struct.pack('>i', 1),                      # Partition 1
            struct.pack('>i', len(msg_c_binary) + 12), # Size of the incoming message set
            struct.pack('>q', 0),                      # No offset specified
            struct.pack('>i', len(msg_c_binary)),      # Length of message
            msg_c_binary,                              # Actual message
        ])

        expected1 = b"".join([ header, topic1, topic2 ])
        expected2 = b"".join([ header, topic2, topic1 ])

        encoded = KafkaProtocol.encode_produce_request(b"client1", 2, requests, 2, 100)
        self.assertIn(encoded, [ expected1, expected2 ])
示例#3
0
    def test_encode_message(self):
        message = create_message("test", "key")
        encoded = KafkaProtocol._encode_message(message)
        expect = "".join([
            struct.pack(">i", -1427009701),  # CRC
            struct.pack(">bb", 0, 0),  # Magic, flags
            struct.pack(">i", 3),  # Length of key
            "key",  # key
            struct.pack(">i", 4),  # Length of value
            "test",  # value
        ])

        self.assertEqual(encoded, expect)
    def test_encode_message(self):
        message = create_message(b"test", b"key")
        encoded = KafkaProtocol._encode_message(message)
        expect = b"".join([
            struct.pack(">i", -1427009701), # CRC
            struct.pack(">bb", 0, 0),       # Magic, flags
            struct.pack(">i", 3),           # Length of key
            b"key",                          # key
            struct.pack(">i", 4),           # Length of value
            b"test",                         # value
        ])

        self.assertEqual(encoded, expect)
示例#5
0
 def test_encode_message_failure(self):
     with self.assertRaises(ProtocolError):
         KafkaProtocol._encode_message(Message(1, 0, "key", "test"))
 def test_encode_message_failure(self):
     with self.assertRaises(ProtocolError):
         KafkaProtocol._encode_message(Message(1, 0, "key", "test"))
示例#7
0
 def test_encode_message(self):
     message = create_message("test", "key")
     encoded = KafkaProtocol._encode_message(message)
     expect = "\xaa\xf1\x8f[\x00\x00\x00\x00\x00\x03key\x00\x00\x00\x04test"
     self.assertEqual(encoded, expect)
示例#8
0
 def test_encode_message(self):
     message = create_message("test", "key")
     encoded = KafkaProtocol._encode_message(message)
     expect = "\xaa\xf1\x8f[\x00\x00\x00\x00\x00\x03key\x00\x00\x00\x04test"
     self.assertEqual(encoded, expect)