def broker_partitions_for(self, topic):
"""Return a list of BrokerPartitions based on values found in
ZooKeeper."""
# Get the broker_ids first...
broker_ids = self.broker_ids_for(topic)
# log.debug(u"broker_ids: {0}".format(broker_ids))
# Then the broker_strings for each broker
broker_paths = map(self.path_for_broker, broker_ids)
# log.debug(u"broker_paths: {0}".format(broker_paths))
broker_strings = map(self._zk_properties, broker_paths)
# log.debug(u"broker_strings: {0}".format(broker_strings))
# Then the num_parts per broker (each could be set differently)
broker_topic_paths = [self.path_for_broker_topic(broker_id, topic)
for broker_id in broker_ids]
num_parts = map(self._zk_properties, broker_topic_paths)
# log.debug(u"num_parts: {0}".format(num_parts))
# BrokerPartition
return list(
chain.from_iterable(
BrokerPartition.from_zk(broker_id, broker_string, topic, n)
for broker_id, broker_string, n
in zip(broker_ids, broker_strings, num_parts)
)
)
def broker_partitions_for(self, topic):
"""Return a list of BrokerPartitions based on values found in
ZooKeeper."""
# Get the broker_ids first...
broker_ids = self.broker_ids_for(topic)
# log.debug(u"broker_ids: {0}".format(broker_ids))
# Then the broker_strings for each broker
broker_paths = map(self.path_for_broker, broker_ids)
# log.debug(u"broker_paths: {0}".format(broker_paths))
broker_strings = map(self._zk_properties, broker_paths)
# log.debug(u"broker_strings: {0}".format(broker_strings))
# Then the num_parts per broker (each could be set differently)
broker_topic_paths = [
self.path_for_broker_topic(broker_id, topic)
for broker_id in broker_ids
]
num_parts = map(self._zk_properties, broker_topic_paths)
# log.debug(u"num_parts: {0}".format(num_parts))
# BrokerPartition
return list(
chain.from_iterable(
BrokerPartition.from_zk(broker_id, broker_string, topic, n)
for broker_id, broker_string, n in zip(
broker_ids, broker_strings, num_parts)))
def broker_partitions_for(self, topic, force_partition_zero=False):
"""Return a list of BrokerPartitions based on values found in
ZooKeeper.
If you set force_partition_zero=True, we will always return partition
0 of a broker, even if no topic has been created for it yet. Consumers
don't need this, because it means there's nothing there to read anyway.
But Producers need to be bootstrapped with it.
"""
# Get the broker_ids first...
if force_partition_zero:
broker_ids = self.all_broker_ids()
else:
broker_ids = self.broker_ids_for(topic)
# log.debug(u"broker_ids: {0}".format(broker_ids))
# Then the broker_strings for each broker
broker_paths = map(self.path_for_broker, broker_ids)
# log.debug(u"broker_paths: {0}".format(broker_paths))
broker_strings = map(self._zk_properties, broker_paths)
# log.debug(u"broker_strings: {0}".format(broker_strings))
# Then the num_parts per broker (each could be set differently)
broker_topic_paths = [self.path_for_broker_topic(broker_id, topic)
for broker_id in broker_ids]
# Every broker has at least one partition, even if it's not published
num_parts = []
for p in broker_topic_paths:
broker_parts = self._zk_properties(p) if self._zk.exists(p) else 1
num_parts.append(broker_parts)
# log.debug(u"num_parts: {0}".format(num_parts))
# BrokerPartition
return list(
chain.from_iterable(
BrokerPartition.from_zk(broker_id, broker_string, topic, n)
for broker_id, broker_string, n
in zip(broker_ids, broker_strings, num_parts)
)
)
def broker_partitions_for(self, topic, force_partition_zero=False):
"""Return a list of BrokerPartitions based on values found in
ZooKeeper.
If you set force_partition_zero=True, we will always return partition
0 of a broker, even if no topic has been created for it yet. Consumers
don't need this, because it means there's nothing there to read anyway.
But Producers need to be bootstrapped with it.
"""
# Get the broker_ids first...
if force_partition_zero:
broker_ids = self.all_broker_ids()
else:
broker_ids = self.broker_ids_for(topic)
# log.debug(u"broker_ids: {0}".format(broker_ids))
# Then the broker_strings for each broker
broker_paths = map(self.path_for_broker, broker_ids)
# log.debug(u"broker_paths: {0}".format(broker_paths))
broker_strings = map(self._zk_properties, broker_paths)
# log.debug(u"broker_strings: {0}".format(broker_strings))
# Then the num_parts per broker (each could be set differently)
broker_topic_paths = [
self.path_for_broker_topic(broker_id, topic)
for broker_id in broker_ids
]
# Every broker has at least one partition, even if it's not published
num_parts = []
for p in broker_topic_paths:
broker_parts = self._zk_properties(p) if self._zk.exists(p) else 1
num_parts.append(broker_parts)
# log.debug(u"num_parts: {0}".format(num_parts))
# BrokerPartition
return list(
chain.from_iterable(
BrokerPartition.from_zk(broker_id, broker_string, topic, n)
for broker_id, broker_string, n in zip(
broker_ids, broker_strings, num_parts)))
