class KafkaIntegrationTestCase(unittest2.TestCase): create_client = True topic = None @deferred(timeout=10) @inlineCallbacks def setUp(self): super(KafkaIntegrationTestCase, self).setUp() if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.error('KAFKA_VERSION unset!') return if not self.topic: self.topic = "%s-%s" % ( self.id()[self.id().rindex(".") + 1:], random_string(10)) if self.create_client: self.client = KafkaClient( '%s:%d' % (self.server.host, self.server.port), clientId=self.topic) yield ensure_topic_creation(self.client, self.topic, reactor=self.reactor) self._messages = {} @deferred(timeout=10) @inlineCallbacks def tearDown(self): super(KafkaIntegrationTestCase, self).tearDown() if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.error('KAFKA_VERSION unset!') return if self.create_client: yield self.client.close() # Check for outstanding delayedCalls. Note, this may yield # spurious errors if the class's client has an outstanding # delayed call due to reconnecting. dcs = self.reactor.getDelayedCalls() if dcs: # pragma: no cover log.error("Outstanding Delayed Calls at tearDown: %s\n\n", ' '.join([str(dc) for dc in dcs])) self.assertFalse(dcs) @inlineCallbacks def current_offset(self, topic, partition): offsets, = yield self.client.send_offset_request( [OffsetRequest(topic, partition, -1, 1)]) returnValue(offsets.offsets[0]) def msg(self, s): if s not in self._messages: self._messages[s] = '%s-%s-%s' % (s, self.id(), str(uuid.uuid4())) return self._messages[s]
class IntegrationMixin(object): """ Mixin for tests that require a Kafka cluster. The `setUp()` and `tearDown()` methods bring up a Kafka cluster and create a topic for the test to use. Mix this into a subclass of `twisted.trial.unittest.TestCase`. Note that you must override *harness_kw* in the subclass. :data dict harness_kw: Keyword arguments for `harness`. Subclasses must set this to specify ``replicas`` (the number of Kafka brokers) and may specify other arguments — see `afkak.fixtures.KafkaHarness.start()`. :data dict client_kw: Keyword arguments for `client`. Subclasses may inject keyword arguments by overriding this. The default is empty. :ivar str topic: Kafka topic name. This may be set in subclasses. If ``None``, a random topic name is generated by `setUp()`. :ivar harness: `afkak.test.fixtures.KafkaHarness` instance. This is created by the `setUp()` method and automatically torn down. :ivar client: `afkak.KafkaClient` instance created by the `setUp()` method. :ivar reactor: Twisted reactor. """ topic = None from twisted.internet import reactor client_kw = {} if not os.environ.get('KAFKA_VERSION'): # pragma: no cover skip = 'KAFKA_VERSION is not set' @inlineCallbacks def setUp(self): log.info("Setting up test %s", self.id()) self.harness = KafkaHarness.start(**self.harness_kw) self.addCleanup(self.harness.halt) if not self.topic: self.topic = "%s-%s" % ( self.id()[self.id().rindex(".") + 1:], random_string(10)) self.client = KafkaClient( self.harness.bootstrap_hosts, clientId=self.__class__.__name__, **self.client_kw ) self.addCleanup(self.client.close) yield ensure_topic_creation(self.client, self.topic, fully_replicated=True) self._messages = {} def tearDown(self): log.info("Tearing down test: %r", self) @inlineCallbacks def current_offset(self, topic, partition): offsets, = yield self.client.send_offset_request( [OffsetRequest(topic, partition, -1, 1)]) returnValue(offsets.offsets[0]) @inlineCallbacks def retry_while_broker_errors(self, f, *a, **kw): """ Call a function, retrying on retriable broker errors. If calling the function fails with one of these exception types it is called again after a short delay: * `afkak.common.RetriableBrokerResponseError` (or a subclass thereof) * `afkak.common.PartitionUnavailableError` The net effect is to keep trying until topic auto-creation completes. :param f: callable, which may return a `Deferred` :param a: arbitrary positional arguments :param kw: arbitrary keyword arguments """ while True: try: returnValue((yield f(*a, **kw))) break except (RetriableBrokerResponseError, PartitionUnavailableError): yield async_delay(0.1, clock=self.reactor) def msg(self, s): if s not in self._messages: self._messages[s] = (u'%s-%s-%s' % (s, self.id(), uuid.uuid4())).encode('utf-8') return self._messages[s]
class KafkaIntegrationTestCase(unittest.TestCase): create_client = True topic = None server = None reactor = None def shortDescription(self): """ Show the ID of the test when nose displays its name, rather than a snippet of the docstring. """ return self.id() @deferred(timeout=10) @inlineCallbacks def setUp(self): log.info("Setting up test %s", self.id()) super(KafkaIntegrationTestCase, self).setUp() if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.error('KAFKA_VERSION unset!') return if not self.topic: self.topic = "%s-%s" % (self.id()[self.id().rindex(".") + 1:], random_string(10)) if self.create_client: self.client = KafkaClient('%s:%d' % (self.server.host, self.server.port), clientId=self.topic) yield ensure_topic_creation(self.client, self.topic, fully_replicated=True, reactor=self.reactor) self._messages = {} @deferred(timeout=10) @inlineCallbacks def tearDown(self): log.info("Tearing down test: %r", self) super(KafkaIntegrationTestCase, self).tearDown() if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.error('KAFKA_VERSION unset!') return if self.create_client: yield self.client.close() # Check for outstanding delayedCalls. Note, this may yield # spurious errors if the class's client has an outstanding # delayed call due to reconnecting. dcs = self.reactor.getDelayedCalls() if dcs: # pragma: no cover log.error("Outstanding Delayed Calls at tearDown: %s\n\n", ' '.join([str(dc) for dc in dcs])) self.assertFalse(dcs) @inlineCallbacks def current_offset(self, topic, partition): offsets, = yield self.client.send_offset_request( [OffsetRequest(topic, partition, -1, 1)]) returnValue(offsets.offsets[0]) def msg(self, s): if s not in self._messages: self._messages[s] = (u'%s-%s-%s' % (s, self.id(), uuid.uuid4())).encode('utf-8') return self._messages[s]