def test_drain_on_list(self):
data = ["foo", 1, "bar", 9]
result = list(iterables.drain(data))
self.assertEqual(len(data), 0)
self.assertEqual(result, [9, "bar", 1, "foo"])
def test_drain_on_set(self):
data = set(["foo", 1, "bar", 9])
result = list(iterables.drain(data))
self.assertEqual(len(data), 0)
self.assertEqual(set(result), set(["foo", 1, "bar", 9]))
def test_drain_on_dict(self):
data = {"foo": 1, "bar": 9}
result = {key: value for key, value in iterables.drain(data)}
self.assertEqual(len(data), 0)
self.assertEqual(result, {"foo": 1, "bar": 9})
def test_drain_on_deque(self):
data = collections.deque(["foo", 1, "bar", 9])
result = list(iterables.drain(data))
self.assertEqual(len(data), 0)
self.assertEqual(result, ["foo", 1, "bar", 9])
def test_drain_on_list(self):
data = ["foo", 1, "bar", 9]
result = list(iterables.drain(data))
self.assertEqual(len(data), 0)
self.assertEqual(result, [9, "bar", 1, "foo"])
def test_drain_on_dict(self):
data = {"foo": 1, "bar": 9}
result = {key: value for key, value in iterables.drain(data)}
self.assertEqual(len(data), 0)
self.assertEqual(result, {"foo": 1, "bar": 9})
def test_drain_on_set(self):
data = set(["foo", 1, "bar", 9])
result = list(iterables.drain(data))
self.assertEqual(len(data), 0)
self.assertEqual(set(result), set(["foo", 1, "bar", 9]))
def test_drain_on_deque(self):
data = collections.deque(["foo", 1, "bar", 9])
result = list(iterables.drain(data))
self.assertEqual(len(data), 0)
self.assertEqual(result, ["foo", 1, "bar", 9])
def flush(self):
"""
Transforms the ``unsent`` structure to produce requests and sends them.
The first order of business is to order the pending messages in
``unsent`` based on partition leader. If a message's partition leader
is not a know broker, the message is queued up to be retried and the
flag denoting that a cluster ``heal()`` call is needed is set.
Once the legitimate messages are ordered, instances of ProduceRequest
are created for each broker and sent.
"""
if not self.unsent:
return
# leader -> topic -> partition -> message list
ordered = collections.defaultdict(lambda: collections.defaultdict(
lambda: collections.defaultdict(list)))
to_retry = collections.defaultdict(list)
for topic, msgs in drain(self.unsent):
for msg in msgs:
partition = self.partitioner(msg.key,
self.cluster.topics[topic])
leader = self.cluster.get_leader(topic, partition)
if leader not in self.cluster:
to_retry[topic].append(msg)
continue
ordered[leader][topic][partition].append(msg)
requests = {}
for leader, topics in six.iteritems(ordered):
requests[leader] = produce_api.ProduceRequest(
required_acks=self.required_acks,
timeout=self.ack_timeout,
topics=[])
for topic, partitions in six.iteritems(topics):
requests[leader].topics.append(
produce_api.TopicRequest(name=topic, partitions=[]))
for partition_id, msgs in six.iteritems(partitions):
requests[leader].topics[-1].partitions.append(
produce_api.PartitionRequest(
partition_id=partition_id,
message_set=messages.MessageSet.compressed(
self.compression, msgs)))
self.sent[requests[leader].
correlation_id][topic][partition_id] = msgs
for topic, msgs in six.iteritems(to_retry):
self.queue_retries(topic, msgs)
yield self.send(requests)
def flush(self):
"""
Transforms the ``unsent`` structure to produce requests and sends them.
The first order of business is to order the pending messages in
``unsent`` based on partition leader. If a message's partition leader
is not a know broker, the message is queued up to be retried and the
flag denoting that a cluster ``heal()`` call is needed is set.
Once the legitimate messages are ordered, instances of ProduceRequest
are created for each broker and sent.
"""
if not self.unsent:
return
# leader -> topic -> partition -> message list
ordered = collections.defaultdict(lambda: collections.defaultdict(lambda: collections.defaultdict(list)))
to_retry = collections.defaultdict(list)
for topic, msgs in drain(self.unsent):
for msg in msgs:
partition = self.partitioner(msg.key, self.cluster.topics[topic])
leader = self.cluster.get_leader(topic, partition)
if leader not in self.cluster:
to_retry[topic].append(msg)
continue
ordered[leader][topic][partition].append(msg)
requests = {}
for leader, topics in six.iteritems(ordered):
requests[leader] = produce_api.ProduceRequest(
required_acks=self.required_acks, timeout=self.ack_timeout, topics=[]
)
for topic, partitions in six.iteritems(topics):
requests[leader].topics.append(produce_api.TopicRequest(name=topic, partitions=[]))
for partition_id, msgs in six.iteritems(partitions):
requests[leader].topics[-1].partitions.append(
produce_api.PartitionRequest(
partition_id=partition_id,
message_set=messages.MessageSet.compressed(self.compression, msgs),
)
)
self.sent[requests[leader].correlation_id][topic][partition_id] = msgs
for topic, msgs in six.iteritems(to_retry):
self.queue_retries(topic, msgs)
yield self.send(requests)
