class KafkaIntegrationTestCase(unittest2.TestCase):
create_client = True
topic = None
@deferred(timeout=10)
@inlineCallbacks
def setUp(self):
super(KafkaIntegrationTestCase, self).setUp()
if not os.environ.get('KAFKA_VERSION'): # pragma: no cover
log.error('KAFKA_VERSION unset!')
return
if not self.topic:
self.topic = "%s-%s" % (
self.id()[self.id().rindex(".") + 1:], random_string(10))
if self.create_client:
self.client = KafkaClient(
'%s:%d' % (self.server.host, self.server.port),
clientId=self.topic)
yield ensure_topic_creation(self.client, self.topic,
reactor=self.reactor)
self._messages = {}
@deferred(timeout=10)
@inlineCallbacks
def tearDown(self):
super(KafkaIntegrationTestCase, self).tearDown()
if not os.environ.get('KAFKA_VERSION'): # pragma: no cover
log.error('KAFKA_VERSION unset!')
return
if self.create_client:
yield self.client.close()
# Check for outstanding delayedCalls. Note, this may yield
# spurious errors if the class's client has an outstanding
# delayed call due to reconnecting.
dcs = self.reactor.getDelayedCalls()
if dcs: # pragma: no cover
log.error("Outstanding Delayed Calls at tearDown: %s\n\n",
' '.join([str(dc) for dc in dcs]))
self.assertFalse(dcs)
@inlineCallbacks
def current_offset(self, topic, partition):
offsets, = yield self.client.send_offset_request(
[OffsetRequest(topic, partition, -1, 1)])
returnValue(offsets.offsets[0])
def msg(self, s):
if s not in self._messages:
self._messages[s] = '%s-%s-%s' % (s, self.id(), str(uuid.uuid4()))
return self._messages[s]
class IntegrationMixin(object):
"""
Mixin for tests that require a Kafka cluster.
The `setUp()` and `tearDown()` methods bring up a Kafka cluster and create
a topic for the test to use.
Mix this into a subclass of `twisted.trial.unittest.TestCase`. Note that
you must override *harness_kw* in the subclass.
:data dict harness_kw:
Keyword arguments for `harness`. Subclasses must set this to specify
``replicas`` (the number of Kafka brokers) and may specify other
arguments — see `afkak.fixtures.KafkaHarness.start()`.
:data dict client_kw:
Keyword arguments for `client`. Subclasses may inject keyword arguments
by overriding this. The default is empty.
:ivar str topic:
Kafka topic name. This may be set in subclasses. If ``None``, a random
topic name is generated by `setUp()`.
:ivar harness:
`afkak.test.fixtures.KafkaHarness` instance. This is created by the
`setUp()` method and automatically torn down.
:ivar client:
`afkak.KafkaClient` instance created by the `setUp()` method.
:ivar reactor: Twisted reactor.
"""
topic = None
from twisted.internet import reactor
client_kw = {}
if not os.environ.get('KAFKA_VERSION'): # pragma: no cover
skip = 'KAFKA_VERSION is not set'
@inlineCallbacks
def setUp(self):
log.info("Setting up test %s", self.id())
self.harness = KafkaHarness.start(**self.harness_kw)
self.addCleanup(self.harness.halt)
if not self.topic:
self.topic = "%s-%s" % (
self.id()[self.id().rindex(".") + 1:], random_string(10))
self.client = KafkaClient(
self.harness.bootstrap_hosts,
clientId=self.__class__.__name__,
**self.client_kw
)
self.addCleanup(self.client.close)
yield ensure_topic_creation(self.client, self.topic,
fully_replicated=True)
self._messages = {}
def tearDown(self):
log.info("Tearing down test: %r", self)
@inlineCallbacks
def current_offset(self, topic, partition):
offsets, = yield self.client.send_offset_request(
[OffsetRequest(topic, partition, -1, 1)])
returnValue(offsets.offsets[0])
@inlineCallbacks
def retry_while_broker_errors(self, f, *a, **kw):
"""
Call a function, retrying on retriable broker errors.
If calling the function fails with one of these exception types it is
called again after a short delay:
* `afkak.common.RetriableBrokerResponseError` (or a subclass thereof)
* `afkak.common.PartitionUnavailableError`
The net effect is to keep trying until topic auto-creation completes.
:param f: callable, which may return a `Deferred`
:param a: arbitrary positional arguments
:param kw: arbitrary keyword arguments
"""
while True:
try:
returnValue((yield f(*a, **kw)))
break
except (RetriableBrokerResponseError, PartitionUnavailableError):
yield async_delay(0.1, clock=self.reactor)
def msg(self, s):
if s not in self._messages:
self._messages[s] = (u'%s-%s-%s' % (s, self.id(), uuid.uuid4())).encode('utf-8')
return self._messages[s]
class KafkaIntegrationTestCase(unittest.TestCase):
create_client = True
topic = None
server = None
reactor = None
def shortDescription(self):
"""
Show the ID of the test when nose displays its name, rather than
a snippet of the docstring.
"""
return self.id()
@deferred(timeout=10)
@inlineCallbacks
def setUp(self):
log.info("Setting up test %s", self.id())
super(KafkaIntegrationTestCase, self).setUp()
if not os.environ.get('KAFKA_VERSION'): # pragma: no cover
log.error('KAFKA_VERSION unset!')
return
if not self.topic:
self.topic = "%s-%s" % (self.id()[self.id().rindex(".") + 1:],
random_string(10))
if self.create_client:
self.client = KafkaClient('%s:%d' %
(self.server.host, self.server.port),
clientId=self.topic)
yield ensure_topic_creation(self.client,
self.topic,
fully_replicated=True,
reactor=self.reactor)
self._messages = {}
@deferred(timeout=10)
@inlineCallbacks
def tearDown(self):
log.info("Tearing down test: %r", self)
super(KafkaIntegrationTestCase, self).tearDown()
if not os.environ.get('KAFKA_VERSION'): # pragma: no cover
log.error('KAFKA_VERSION unset!')
return
if self.create_client:
yield self.client.close()
# Check for outstanding delayedCalls. Note, this may yield
# spurious errors if the class's client has an outstanding
# delayed call due to reconnecting.
dcs = self.reactor.getDelayedCalls()
if dcs: # pragma: no cover
log.error("Outstanding Delayed Calls at tearDown: %s\n\n",
' '.join([str(dc) for dc in dcs]))
self.assertFalse(dcs)
@inlineCallbacks
def current_offset(self, topic, partition):
offsets, = yield self.client.send_offset_request(
[OffsetRequest(topic, partition, -1, 1)])
returnValue(offsets.offsets[0])
def msg(self, s):
if s not in self._messages:
self._messages[s] = (u'%s-%s-%s' %
(s, self.id(), uuid.uuid4())).encode('utf-8')
return self._messages[s]