def test_send_produce_request_maintains_request_response_order(self):
self.client.ensure_topic_exists(b'foo')
self.client.ensure_topic_exists(b'bar')
requests = [
ProduceRequest(b'foo', 0,
[create_message(b'a'),
create_message(b'b')]),
ProduceRequest(b'bar', 1,
[create_message(b'a'),
create_message(b'b')]),
ProduceRequest(b'foo', 1,
[create_message(b'a'),
create_message(b'b')]),
ProduceRequest(b'bar', 0,
[create_message(b'a'),
create_message(b'b')]),
]
responses = self.client.send_produce_request(requests)
while len(responses):
request = requests.pop()
response = responses.pop()
self.assertEqual(request.topic, response.topic)
self.assertEqual(request.partition, response.partition)
def test_encode_produce_request(self):
requests = [
ProduceRequest(
"topic1", 0,
[create_message("a"), create_message("b")]),
ProduceRequest("topic2", 1, [create_message("c")])
]
msg_a_binary = KafkaProtocol._encode_message(create_message("a"))
msg_b_binary = KafkaProtocol._encode_message(create_message("b"))
msg_c_binary = KafkaProtocol._encode_message(create_message("c"))
header = "".join([
struct.pack('>i', 0x94), # The length of the message overall
struct.pack('>h', 0), # Msg Header, Message type = Produce
struct.pack('>h', 0), # Msg Header, API version
struct.pack('>i', 2), # Msg Header, Correlation ID
struct.pack('>h7s', 7, "client1"), # Msg Header, The client ID
struct.pack('>h', 2), # Num acks required
struct.pack('>i', 100), # Request Timeout
struct.pack('>i', 2), # The number of requests
])
total_len = len(msg_a_binary) + len(msg_b_binary)
topic1 = "".join([
struct.pack('>h6s', 6, 'topic1'), # The topic1
struct.pack('>i', 1), # One message set
struct.pack('>i', 0), # Partition 0
struct.pack('>i',
total_len + 24), # Size of the incoming message set
struct.pack('>q', 0), # No offset specified
struct.pack('>i', len(msg_a_binary)), # Length of message
msg_a_binary, # Actual message
struct.pack('>q', 0), # No offset specified
struct.pack('>i', len(msg_b_binary)), # Length of message
msg_b_binary, # Actual message
])
topic2 = "".join([
struct.pack('>h6s', 6, 'topic2'), # The topic1
struct.pack('>i', 1), # One message set
struct.pack('>i', 1), # Partition 1
struct.pack('>i',
len(msg_c_binary) +
12), # Size of the incoming message set
struct.pack('>q', 0), # No offset specified
struct.pack('>i', len(msg_c_binary)), # Length of message
msg_c_binary, # Actual message
])
expected1 = "".join([header, topic1, topic2])
expected2 = "".join([header, topic2, topic1])
encoded = KafkaProtocol.encode_produce_request("client1", 2, requests,
2, 100)
self.assertIn(encoded, [expected1, expected2])
def test_send_produce_request_raises_when_topic_unknown(self, protocol):
@asyncio.coroutine
def recv(request_id):
return b'response'
mocked_conns = {('broker_1', 4567): mock.MagicMock()}
mocked_conns[('broker_1', 4567)].recv.side_effect = recv
client = AIOKafkaClient(['broker_1:4567'], loop=self.loop)
client._conns = mocked_conns
brokers = [
BrokerMetadata(0, 'broker_1', 4567),
BrokerMetadata(1, 'broker_2', 5678)
]
topics = [
TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION,
[]),
]
protocol.decode_metadata_response.return_value = MetadataResponse(
brokers, topics)
self.loop.run_until_complete(client.load_metadata_for_topics())
requests = [
ProduceRequest(
"topic_doesnt_exist", 0,
[create_message("a"), create_message("b")])
]
with self.assertRaises(UnknownTopicOrPartitionError):
self.loop.run_until_complete(client.send_produce_request(requests))
def test_send_produce_request_raises_when_noleader(self, protocol, conn):
"Send producer request raises LeaderUnavailableError if leader is not available"
conn.recv.return_value = 'response' # anything but None
brokers = {}
brokers[0] = BrokerMetadata(0, 'broker_1', 4567)
brokers[1] = BrokerMetadata(1, 'broker_2', 5678)
topics = {}
topics['topic_noleader'] = {
0: PartitionMetadata('topic_noleader', 0, -1, [], []),
1: PartitionMetadata('topic_noleader', 1, -1, [], [])
}
protocol.decode_metadata_response.return_value = (brokers, topics)
client = KafkaClient(hosts=['broker_1:4567'])
requests = [
ProduceRequest(
"topic_noleader", 0,
[create_message("a"), create_message("b")])
]
with self.assertRaises(LeaderUnavailableError):
client.send_produce_request(requests)
def test_send_produce_request_raises_when_noleader(self, protocol, conn):
"Send producer request raises LeaderNotAvailableError if leader is not available"
conn.recv.return_value = 'response' # anything but None
brokers = [
BrokerMetadata(0, 'broker_1', 4567),
BrokerMetadata(1, 'broker_2', 5678)
]
topics = [
TopicMetadata('topic_noleader', NO_ERROR, [
PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER),
PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER),
]),
]
protocol.decode_metadata_response.return_value = MetadataResponse(
brokers, topics)
client = KafkaClient(hosts=['broker_1:4567'])
requests = [
ProduceRequest(
"topic_noleader", 0,
[create_message("a"), create_message("b")])
]
with self.assertRaises(LeaderNotAvailableError):
client.send_produce_request(requests)
def send_messages(self, partition, messages):
messages = [ create_message(self.msg(str(msg))) for msg in messages ]
produce = ProduceRequest(self.topic, partition, messages = messages)
resp, = self.client.send_produce_request([produce])
self.assertEqual(resp.error, 0)
return [ x.value for x in messages ]
def test_send_produce_request_raises_when_topic_unknown(
self, protocol, conn):
conn.recv.return_value = 'response' # anything but None
brokers = [
BrokerMetadata(0, 'broker_1', 4567),
BrokerMetadata(1, 'broker_2', 5678)
]
topics = [
TopicMetadata('topic_doesnt_exist', UNKNOWN_TOPIC_OR_PARTITION,
[]),
]
protocol.decode_metadata_response.return_value = MetadataResponse(
brokers, topics)
client = KafkaClient(hosts=['broker_1:4567'])
requests = [
ProduceRequest(
"topic_doesnt_exist", 0,
[create_message("a"), create_message("b")])
]
with self.assertRaises(UnknownTopicOrPartitionError):
client.send_produce_request(requests)
def test_send_without_response(self):
"""Imitate producer without acknowledge, in this case client produces
messages and kafka does not send response, and we make sure that
futures do not stuck in queue forever"""
host, port = self.kafka_host, self.kafka_port
conn = yield from create_conn(host, port, loop=self.loop)
# prepare message
msgs = create_message_set([b'foo'], 0, None)
req = ProduceRequest(b'bar', 0, msgs)
encoder = functools.partial(KafkaProtocol.encode_produce_request,
acks=0,
timeout=int(10 * 1000))
request_id = 1
client_id = b"aiokafka-python"
request = encoder(client_id=client_id,
correlation_id=request_id,
payloads=[req])
# produce messages without acknowledge
for i in range(100):
conn.send(request, no_ack=True)
# make sure futures no stuck in queue
self.assertEqual(len(conn._requests), 0)
def assert_produce_request(self, messages, initial_offset, message_ct):
produce = ProduceRequest(self.topic, 0, messages=messages)
# There should only be one response message from the server.
# This will throw an exception if there's more than one.
resp = yield from self.client.send_produce_request([produce])
self.assert_produce_response(resp, initial_offset)
offset = yield from self.current_offset(self.topic, 0)
self.assertEqual(offset, initial_offset + message_ct)
def assert_last_retry_result(self, last_retry_result, message,
expected_published_msgs_count):
expected_requests = [
ProduceRequest(
topic=message.topic,
partition=0,
messages=[_prepare(_EnvelopeAndMessage(Envelope(), message))])
]
assert last_retry_result.unpublished_requests == expected_requests
assert last_retry_result.total_published_message_count == expected_published_msgs_count
def test_encode_produce_request(self):
requests = [
ProduceRequest(
"topic1", 0,
[create_message("a"), create_message("b")]),
ProduceRequest("topic2", 1, [create_message("c")])
]
expect = ('\x00\x00\x00\x94\x00\x00\x00\x00\x00\x00\x00\x02\x00\x07'
'client1\x00\x02\x00\x00\x00d\x00\x00\x00\x02\x00\x06topic1'
'\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x006\x00\x00\x00'
'\x00\x00\x00\x00\x00\x00\x00\x00\x0fQ\xdf:2\x00\x00\xff\xff'
'\xff\xff\x00\x00\x00\x01a\x00\x00\x00\x00\x00\x00\x00\x00'
'\x00\x00\x00\x0f\xc8\xd6k\x88\x00\x00\xff\xff\xff\xff\x00'
'\x00\x00\x01b\x00\x06topic2\x00\x00\x00\x01\x00\x00\x00\x01'
'\x00\x00\x00\x1b\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
'\x00\x0f\xbf\xd1[\x1e\x00\x00\xff\xff\xff\xff\x00\x00\x00'
'\x01c')
encoded = KafkaProtocol.encode_produce_request("client1", 2, requests,
2, 100)
self.assertEqual(encoded, expect)
def produce_messages(self):
"""
Produce sample messages
"""
# TODO: Support different kafka port
kafka = KafkaClient(self.config.kafka_host)
total_messages = self.batches * self.batch_size
messages_batch = [
create_message(random.choice(self.sample_messages))
for _ in xrange(self.batch_size)
]
for i in range(self.batches):
# TODO: Support writing to all partitions
req = ProduceRequest(topic=self.config.kafka_topic,
partition=0,
messages=messages_batch)
kafka.send_produce_request(payloads=[req], fail_on_error=True)
sent_messages = i * self.batch_size
logging.info('Created %s out of %s sample messages', sent_messages,
total_messages)
kafka.close()
def test_send_produce_request_raises_when_noleader(self, protocol):
"""Send producer request raises LeaderNotAvailableError
if leader is not available"""
@asyncio.coroutine
def recv(request_id):
return b'response'
mocked_conns = {('broker_1', 4567): mock.MagicMock()}
mocked_conns[('broker_1', 4567)].recv.side_effect = recv
client = AIOKafkaClient(['broker_1:4567'], loop=self.loop)
client._conns = mocked_conns
brokers = [
BrokerMetadata(0, 'broker_1', 4567),
BrokerMetadata(1, 'broker_2', 5678)
]
topics = [
TopicMetadata('topic_noleader', NO_ERROR, [
PartitionMetadata('topic_noleader', 0, -1, [], [], NO_LEADER),
PartitionMetadata('topic_noleader', 1, -1, [], [], NO_LEADER),
]),
]
protocol.decode_metadata_response.return_value = MetadataResponse(
brokers, topics)
self.loop.run_until_complete(client.load_metadata_for_topics())
requests = [
ProduceRequest(
"topic_noleader", 0,
[create_message("a"), create_message("b")])
]
with self.assertRaises(LeaderNotAvailableError):
self.loop.run_until_complete(client.send_produce_request(requests))
logger.info("loading from: %s", datetime.utcfromtimestamp(last_ts))
stream.add_filter('record-type', 'ribs')
stream.add_filter('record-type', 'updates')
stream.add_interval_filter(last_ts, 0)
# Start the stream
stream.start()
client = KafkaClient(args.our_servers.split(","))
count = 0
for batch in group_by_n(
messages_from_internal(iterate_stream(stream, args.collector)),
1000):
req = ProduceRequest("rib-{}".format(args.collector), 0, batch)
for msg in reversed(req.messages):
if msg.value is None:
continue
last_timestamp = json.loads(msg.value)["timestamp"]
break
count += len(batch)
logger.info("sending %i", count)
res = client.send_produce_request([req])
try:
# this is a bit buggy but it will do for now
with open(save_file, "w") as f:
f.write(str(last_timestamp))
except:
logger.warning("could not write offsets to %s", save_file)
def test_produce_request(self):
req = ProduceRequest("my-topic", 0,
[KafkaClient.create_message("testing")])
enc = KafkaClient.encode_produce_request(req)
expect = "\x00\x00\x00\x08my-topic\x00\x00\x00\x00\x00\x00\x00\x11\x00\x00\x00\r\x01\x00\xe8\xf3Z\x06testing"
self.assertEquals(enc, expect)
def _generate_produce_requests(self):
return [
ProduceRequest(topic=topic, partition=0, messages=messages)
for topic, messages in self._generate_prepared_topic_and_messages()
]
def _send(self, topic, partition, *msgs, key=None):
messages = create_message_set(msgs, self._codec, key)
req = ProduceRequest(topic, partition, messages)
resp = yield from self._client.send_produce_request(
[req], acks=self._req_acks, ack_timeout=self._ack_timeout)
return resp
relation.labels(args.collector).inc()
filter_out = True
if "direct" in msg:
connected.labels(args.collector).inc()
filter_out = True
if msg.get("caida_private", False) is True:
caida_private.labels(args.collector).inc()
filter_out = True
if msg.get("caida_as2org", False) is True:
caida_as2org.labels(args.collector).inc()
filter_out = True
if msg.get("caida_relation", False) is True:
caida_relation.labels(args.collector).inc()
filter_out = True
if msg.get("caida_cone", False) is True:
caida_cone.labels(args.collector).inc()
filter_out = True
if msg.get("caida_as2rel", False) is True:
caida_as2rel.labels(args.collector).inc()
filter_out = True
all_events.labels(args.collector, filter_out).inc()
if filter_out:
continue
abnormal.labels(args.collector).inc()
client.send_produce_request([ProduceRequest("conflicts", PARTITIONS[args.collector], [create_message(json.dumps(msg))])])
#Listing A.1.5
from kafka import KafkaClient, SimpleProducer
kafka = KafkaClient("localhost:9092")
producer = SimpleProducer(kafka,
async=False,
req_acks=SimpleProducer.ACK_AFTER_CLUSTER_COMMIT,
ack_timeout=2000)
producer.send_messages("test-replicated-topic", "Hello Kafka Cluster!")
producer.send_messages("test-replicated-topic", "Message to be replicated.")
producer.send_messages("test-replicated-topic", "And so is this!")
#Listing A.1.8
from kafka import KafkaClient
from kafka.common import ProduceRequest
from kafka.protocol import KafkaProtocol, create_message
kafka = KafkaClient("localhost:9092")
f = open('A1.data', 'r')
for line in f:
s = line.split("\t")[0]
part = abs(hash(s)) % 3
req = ProduceRequest(topic="click-streams",
partition=part,
messages=[create_message(s)])
resps = kafka.send_produce_request(payloads=[req], fail_on_error=True)
