class KafkaIntegrationTestCase(unittest2.TestCase):
create_client = True
topic = None
@deferred(timeout=10)
@inlineCallbacks
def setUp(self):
super(KafkaIntegrationTestCase, self).setUp()
if not os.environ.get('KAFKA_VERSION'): # pragma: no cover
log.error('KAFKA_VERSION unset!')
return
if not self.topic:
self.topic = "%s-%s" % (
self.id()[self.id().rindex(".") + 1:], random_string(10))
if self.create_client:
self.client = KafkaClient(
'%s:%d' % (self.server.host, self.server.port),
clientId=self.topic)
yield ensure_topic_creation(self.client, self.topic,
reactor=self.reactor)
self._messages = {}
@deferred(timeout=10)
@inlineCallbacks
def tearDown(self):
super(KafkaIntegrationTestCase, self).tearDown()
if not os.environ.get('KAFKA_VERSION'): # pragma: no cover
log.error('KAFKA_VERSION unset!')
return
if self.create_client:
yield self.client.close()
# Check for outstanding delayedCalls. Note, this may yield
# spurious errors if the class's client has an outstanding
# delayed call due to reconnecting.
dcs = self.reactor.getDelayedCalls()
if dcs: # pragma: no cover
log.error("Outstanding Delayed Calls at tearDown: %s\n\n",
' '.join([str(dc) for dc in dcs]))
self.assertFalse(dcs)
@inlineCallbacks
def current_offset(self, topic, partition):
offsets, = yield self.client.send_offset_request(
[OffsetRequest(topic, partition, -1, 1)])
returnValue(offsets.offsets[0])
def msg(self, s):
if s not in self._messages:
self._messages[s] = '%s-%s-%s' % (s, self.id(), str(uuid.uuid4()))
return self._messages[s]
def _count_messages(self, topic):
messages = []
hosts = '%s:%d,%s:%d' % (
self.kafka_brokers[0].host, self.kafka_brokers[0].port,
self.kafka_brokers[1].host, self.kafka_brokers[1].port)
client = KafkaClient(hosts, clientId="CountMessages", timeout=500)
try:
yield ensure_topic_creation(client,
topic,
fully_replicated=False,
reactor=self.reactor)
# Need to retry this until we have a leader...
while True:
# Ask the client to load the latest metadata. This may avoid a
# NotLeaderForPartitionError I was seeing upon re-start of the
# broker.
yield client.load_metadata_for_topics(topic)
# if there is an error on the metadata for the topic, raise
if check_error(client.metadata_error_for_topic(topic),
False) is None:
break
# Ok, should be safe to get the partitions now...
partitions = client.topic_partitions[topic]
requests = [
FetchRequest(topic, part, 0, 1024 * 1024)
for part in partitions
]
resps = []
while not resps:
try:
log.debug("_count_message: Fetching messages")
# Prevent log.error() call from causing test failure
with patch.object(kclient, 'log'):
resps = yield client.send_fetch_request(
requests, max_wait_time=400)
except (NotLeaderForPartitionError,
UnknownTopicOrPartitionError,
KafkaUnavailableError): # pragma: no cover
log.debug("_count_message: Metadata err, retrying...")
yield client.load_metadata_for_topics(topic)
except FailedPayloadsError as e: # pragma: no cover
if not e.args[1][0][1].check(RequestTimedOutError):
raise
log.debug("_count_message: Timed out err, retrying...")
finally:
yield client.close()
for fetch_resp in resps:
messages.extend(list(fetch_resp.messages))
log.debug("Got %d messages:%r", len(messages), messages)
returnValue(len(messages))
def _count_messages(self, topic):
log.debug("Counting messages on topic %s", topic)
messages = []
client = KafkaClient(self.harness.bootstrap_hosts,
clientId="CountMessages",
timeout=500,
reactor=self.reactor)
try:
yield ensure_topic_creation(client, topic, fully_replicated=False)
# Need to retry this until we have a leader...
while True:
# Ask the client to load the latest metadata. This may avoid a
# NotLeaderForPartitionError I was seeing upon re-start of the
# broker.
yield client.load_metadata_for_topics(topic)
# if there is an error on the metadata for the topic, wait
errno = client.metadata_error_for_topic(topic)
if errno == 0:
break
else:
log.debug("Topic %s in error errno=%d", topic, errno)
yield async_delay(1.0)
# Ok, should be safe to get the partitions now...
partitions = client.topic_partitions[topic]
requests = [
FetchRequest(topic, part, 0, 1024 * 1024)
for part in partitions
]
resps = []
while not resps:
try:
log.debug("_count_message: Fetching messages")
resps = yield client.send_fetch_request(requests,
max_wait_time=400)
except (NotLeaderForPartitionError,
UnknownTopicOrPartitionError,
KafkaUnavailableError): # pragma: no cover
log.debug("_count_message: Metadata err, retrying...")
yield client.load_metadata_for_topics(topic)
except FailedPayloadsError as e: # pragma: no cover
if not e.args[1][0][1].check(RequestTimedOutError):
raise
log.debug("_count_message: Timed out err, retrying...")
finally:
yield client.close()
for fetch_resp in resps:
messages.extend(list(fetch_resp.messages))
log.debug("Got %d messages: %r", len(messages), messages)
returnValue(len(messages))
def _count_messages(self, topic):
messages = []
hosts = '%s:%d,%s:%d' % (self.kafka_brokers[0].host,
self.kafka_brokers[0].port,
self.kafka_brokers[1].host,
self.kafka_brokers[1].port)
client = KafkaClient(hosts, clientId="CountMessages", timeout=500)
try:
yield ensure_topic_creation(client, topic,
reactor=self.reactor)
# Need to retry this until we have a leader...
while True:
# Ask the client to load the latest metadata. This may avoid a
# NotLeaderForPartitionError I was seeing upon re-start of the
# broker.
yield client.load_metadata_for_topics(topic)
# if there is an error on the metadata for the topic, raise
if check_error(
client.metadata_error_for_topic(topic), False) is None:
break
# Ok, should be safe to get the partitions now...
partitions = client.topic_partitions[topic]
requests = [FetchRequest(topic, part, 0, 1024 * 1024)
for part in partitions]
resps = []
while not resps:
try:
log.debug("_count_message: Fetching messages")
# Prevent log.error() call from causing test failure
with patch.object(kclient, 'log'):
resps = yield client.send_fetch_request(
requests, max_wait_time=400)
except (NotLeaderForPartitionError,
UnknownTopicOrPartitionError,
KafkaUnavailableError): # pragma: no cover
log.debug("_count_message: Metadata err, retrying...")
yield client.load_metadata_for_topics(topic)
except FailedPayloadsError as e: # pragma: no cover
if not e.args[1][0][1].check(RequestTimedOutError):
raise
log.debug("_count_message: Timed out err, retrying...")
finally:
yield client.close()
for fetch_resp in resps:
messages.extend(list(fetch_resp.messages))
log.debug("Got %d messages:%r", len(messages), messages)
returnValue(len(messages))
class KafkaIntegrationTestCase(unittest.TestCase):
create_client = True
topic = None
server = None
reactor = None
def shortDescription(self):
"""
Show the ID of the test when nose displays its name, rather than
a snippet of the docstring.
"""
return self.id()
@deferred(timeout=10)
@inlineCallbacks
def setUp(self):
log.info("Setting up test %s", self.id())
super(KafkaIntegrationTestCase, self).setUp()
if not os.environ.get('KAFKA_VERSION'): # pragma: no cover
log.error('KAFKA_VERSION unset!')
return
if not self.topic:
self.topic = "%s-%s" % (self.id()[self.id().rindex(".") + 1:],
random_string(10))
if self.create_client:
self.client = KafkaClient('%s:%d' %
(self.server.host, self.server.port),
clientId=self.topic)
yield ensure_topic_creation(self.client,
self.topic,
fully_replicated=True,
reactor=self.reactor)
self._messages = {}
@deferred(timeout=10)
@inlineCallbacks
def tearDown(self):
log.info("Tearing down test: %r", self)
super(KafkaIntegrationTestCase, self).tearDown()
if not os.environ.get('KAFKA_VERSION'): # pragma: no cover
log.error('KAFKA_VERSION unset!')
return
if self.create_client:
yield self.client.close()
# Check for outstanding delayedCalls. Note, this may yield
# spurious errors if the class's client has an outstanding
# delayed call due to reconnecting.
dcs = self.reactor.getDelayedCalls()
if dcs: # pragma: no cover
log.error("Outstanding Delayed Calls at tearDown: %s\n\n",
' '.join([str(dc) for dc in dcs]))
self.assertFalse(dcs)
@inlineCallbacks
def current_offset(self, topic, partition):
offsets, = yield self.client.send_offset_request(
[OffsetRequest(topic, partition, -1, 1)])
returnValue(offsets.offsets[0])
def msg(self, s):
if s not in self._messages:
self._messages[s] = (u'%s-%s-%s' %
(s, self.id(), uuid.uuid4())).encode('utf-8')
return self._messages[s]
