def _count_messages(self, topic):
messages = []
hosts = '%s:%d,%s:%d' % (
self.kafka_brokers[0].host, self.kafka_brokers[0].port,
self.kafka_brokers[1].host, self.kafka_brokers[1].port)
client = KafkaClient(hosts, clientId="CountMessages", timeout=500)
try:
yield ensure_topic_creation(client,
topic,
fully_replicated=False,
reactor=self.reactor)
# Need to retry this until we have a leader...
while True:
# Ask the client to load the latest metadata. This may avoid a
# NotLeaderForPartitionError I was seeing upon re-start of the
# broker.
yield client.load_metadata_for_topics(topic)
# if there is an error on the metadata for the topic, raise
if check_error(client.metadata_error_for_topic(topic),
False) is None:
break
# Ok, should be safe to get the partitions now...
partitions = client.topic_partitions[topic]
requests = [
FetchRequest(topic, part, 0, 1024 * 1024)
for part in partitions
]
resps = []
while not resps:
try:
log.debug("_count_message: Fetching messages")
# Prevent log.error() call from causing test failure
with patch.object(kclient, 'log'):
resps = yield client.send_fetch_request(
requests, max_wait_time=400)
except (NotLeaderForPartitionError,
UnknownTopicOrPartitionError,
KafkaUnavailableError): # pragma: no cover
log.debug("_count_message: Metadata err, retrying...")
yield client.load_metadata_for_topics(topic)
except FailedPayloadsError as e: # pragma: no cover
if not e.args[1][0][1].check(RequestTimedOutError):
raise
log.debug("_count_message: Timed out err, retrying...")
finally:
yield client.close()
for fetch_resp in resps:
messages.extend(list(fetch_resp.messages))
log.debug("Got %d messages:%r", len(messages), messages)
returnValue(len(messages))
def _count_messages(self, topic):
log.debug("Counting messages on topic %s", topic)
messages = []
client = KafkaClient(self.harness.bootstrap_hosts,
clientId="CountMessages",
timeout=500,
reactor=self.reactor)
try:
yield ensure_topic_creation(client, topic, fully_replicated=False)
# Need to retry this until we have a leader...
while True:
# Ask the client to load the latest metadata. This may avoid a
# NotLeaderForPartitionError I was seeing upon re-start of the
# broker.
yield client.load_metadata_for_topics(topic)
# if there is an error on the metadata for the topic, wait
errno = client.metadata_error_for_topic(topic)
if errno == 0:
break
else:
log.debug("Topic %s in error errno=%d", topic, errno)
yield async_delay(1.0)
# Ok, should be safe to get the partitions now...
partitions = client.topic_partitions[topic]
requests = [
FetchRequest(topic, part, 0, 1024 * 1024)
for part in partitions
]
resps = []
while not resps:
try:
log.debug("_count_message: Fetching messages")
resps = yield client.send_fetch_request(requests,
max_wait_time=400)
except (NotLeaderForPartitionError,
UnknownTopicOrPartitionError,
KafkaUnavailableError): # pragma: no cover
log.debug("_count_message: Metadata err, retrying...")
yield client.load_metadata_for_topics(topic)
except FailedPayloadsError as e: # pragma: no cover
if not e.args[1][0][1].check(RequestTimedOutError):
raise
log.debug("_count_message: Timed out err, retrying...")
finally:
yield client.close()
for fetch_resp in resps:
messages.extend(list(fetch_resp.messages))
log.debug("Got %d messages: %r", len(messages), messages)
returnValue(len(messages))
def _count_messages(self, topic):
messages = []
hosts = '%s:%d,%s:%d' % (self.kafka_brokers[0].host,
self.kafka_brokers[0].port,
self.kafka_brokers[1].host,
self.kafka_brokers[1].port)
client = KafkaClient(hosts, clientId="CountMessages", timeout=500)
try:
yield ensure_topic_creation(client, topic,
reactor=self.reactor)
# Need to retry this until we have a leader...
while True:
# Ask the client to load the latest metadata. This may avoid a
# NotLeaderForPartitionError I was seeing upon re-start of the
# broker.
yield client.load_metadata_for_topics(topic)
# if there is an error on the metadata for the topic, raise
if check_error(
client.metadata_error_for_topic(topic), False) is None:
break
# Ok, should be safe to get the partitions now...
partitions = client.topic_partitions[topic]
requests = [FetchRequest(topic, part, 0, 1024 * 1024)
for part in partitions]
resps = []
while not resps:
try:
log.debug("_count_message: Fetching messages")
# Prevent log.error() call from causing test failure
with patch.object(kclient, 'log'):
resps = yield client.send_fetch_request(
requests, max_wait_time=400)
except (NotLeaderForPartitionError,
UnknownTopicOrPartitionError,
KafkaUnavailableError): # pragma: no cover
log.debug("_count_message: Metadata err, retrying...")
yield client.load_metadata_for_topics(topic)
except FailedPayloadsError as e: # pragma: no cover
if not e.args[1][0][1].check(RequestTimedOutError):
raise
log.debug("_count_message: Timed out err, retrying...")
finally:
yield client.close()
for fetch_resp in resps:
messages.extend(list(fetch_resp.messages))
log.debug("Got %d messages:%r", len(messages), messages)
returnValue(len(messages))