Exemplo n.º 1
0
def test_encode_message_set():
    messages = [
        Message(b'v1', key=b'k1'),
        Message(b'v2', key=b'k2')
    ]
    encoded = MessageSet.encode([(0, msg.encode())
                                 for msg in messages])
    expect = b''.join([
        struct.pack('>q', 0),          # MsgSet Offset
        struct.pack('>i', 18),         # Msg Size
        struct.pack('>i', 1474775406), # CRC
        struct.pack('>bb', 0, 0),      # Magic, flags
        struct.pack('>i', 2),          # Length of key
        b'k1',                          # Key
        struct.pack('>i', 2),          # Length of value
        b'v1',                          # Value

        struct.pack('>q', 0),          # MsgSet Offset
        struct.pack('>i', 18),         # Msg Size
        struct.pack('>i', -16383415),  # CRC
        struct.pack('>bb', 0, 0),      # Magic, flags
        struct.pack('>i', 2),          # Length of key
        b'k2',                          # Key
        struct.pack('>i', 2),          # Length of value
        b'v2',                          # Value
    ])
    expect = struct.pack('>i', len(expect)) + expect
    assert encoded == expect
Exemplo n.º 2
0
def test_encode_message_set():
    messages = [
        Message(b'v1', key=b'k1'),
        Message(b'v2', key=b'k2')
    ]
    encoded = MessageSet.encode([(0, msg.encode())
                                 for msg in messages],
                                size=False)
    expect = b''.join([
        struct.pack('>q', 0),          # MsgSet Offset
        struct.pack('>i', 18),         # Msg Size
        struct.pack('>i', 1474775406), # CRC
        struct.pack('>bb', 0, 0),      # Magic, flags
        struct.pack('>i', 2),          # Length of key
        b'k1',                          # Key
        struct.pack('>i', 2),          # Length of value
        b'v1',                          # Value

        struct.pack('>q', 0),          # MsgSet Offset
        struct.pack('>i', 18),         # Msg Size
        struct.pack('>i', -16383415),  # CRC
        struct.pack('>bb', 0, 0),      # Magic, flags
        struct.pack('>i', 2),          # Length of key
        b'k2',                          # Key
        struct.pack('>i', 2),          # Length of value
        b'v2',                          # Value
    ])
    assert encoded == expect
Exemplo n.º 3
0
    def encode_produce_request(cls, payloads=(), acks=1, timeout=1000):
        """
        Encode a ProduceRequest struct

        Arguments:
            payloads: list of ProduceRequestPayload
            acks: How "acky" you want the request to be
                1: written to disk by the leader
                0: immediate response
                -1: waits for all replicas to be in sync
            timeout: Maximum time (in ms) the server will wait for replica acks.
                This is _not_ a socket timeout

        Returns: ProduceRequest
        """
        if acks not in (1, 0, -1):
            raise ValueError('ProduceRequest acks (%s) must be 1, 0, -1' %
                             acks)

        topics = []
        for topic, topic_payloads in group_by_topic_and_partition(
                payloads).items():
            topic_msgs = []
            for partition, payload in topic_payloads.items():
                partition_msgs = []
                for msg in payload.messages:
                    m = kafka.protocol.message.Message(
                        msg.value,
                        key=msg.key,
                        magic=msg.magic,
                        attributes=msg.attributes)
                    partition_msgs.append((0, m.encode()))
                topic_msgs.append((partition,
                                   MessageSet.encode(partition_msgs,
                                                     prepend_size=False)))
            topics.append((topic, topic_msgs))

        return kafka.protocol.produce.ProduceRequest[0](required_acks=acks,
                                                        timeout=timeout,
                                                        topics=topics)
Exemplo n.º 4
0
    def encode_produce_request(cls, payloads=(), acks=1, timeout=1000):
        """
        Encode a ProduceRequest struct

        Arguments:
            payloads: list of ProduceRequestPayload
            acks: How "acky" you want the request to be
                1: written to disk by the leader
                0: immediate response
                -1: waits for all replicas to be in sync
            timeout: Maximum time (in ms) the server will wait for replica acks.
                This is _not_ a socket timeout

        Returns: ProduceRequest
        """
        if acks not in (1, 0, -1):
            raise ValueError('ProduceRequest acks (%s) must be 1, 0, -1' % acks)

        topics = []
        for topic, topic_payloads in group_by_topic_and_partition(payloads).items():
            topic_msgs = []
            for partition, payload in topic_payloads.items():
                partition_msgs = []
                for msg in payload.messages:
                    m = kafka.protocol.message.Message(
                          msg.value, key=msg.key,
                          magic=msg.magic, attributes=msg.attributes
                    )
                    partition_msgs.append((0, m.encode()))
                topic_msgs.append((partition, MessageSet.encode(partition_msgs, prepend_size=False)))
            topics.append((topic, topic_msgs))


        return kafka.protocol.produce.ProduceRequest[0](
            required_acks=acks,
            timeout=timeout,
            topics=topics
        )