class KafkaIntegrationTestCase(unittest2.TestCase): create_client = True topic = None @deferred(timeout=10) @inlineCallbacks def setUp(self): super(KafkaIntegrationTestCase, self).setUp() if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.error('KAFKA_VERSION unset!') return if not self.topic: self.topic = "%s-%s" % ( self.id()[self.id().rindex(".") + 1:], random_string(10)) if self.create_client: self.client = KafkaClient( '%s:%d' % (self.server.host, self.server.port), clientId=self.topic) yield ensure_topic_creation(self.client, self.topic, reactor=self.reactor) self._messages = {} @deferred(timeout=10) @inlineCallbacks def tearDown(self): super(KafkaIntegrationTestCase, self).tearDown() if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.error('KAFKA_VERSION unset!') return if self.create_client: yield self.client.close() # Check for outstanding delayedCalls. Note, this may yield # spurious errors if the class's client has an outstanding # delayed call due to reconnecting. dcs = self.reactor.getDelayedCalls() if dcs: # pragma: no cover log.error("Outstanding Delayed Calls at tearDown: %s\n\n", ' '.join([str(dc) for dc in dcs])) self.assertFalse(dcs) @inlineCallbacks def current_offset(self, topic, partition): offsets, = yield self.client.send_offset_request( [OffsetRequest(topic, partition, -1, 1)]) returnValue(offsets.offsets[0]) def msg(self, s): if s not in self._messages: self._messages[s] = '%s-%s-%s' % (s, self.id(), str(uuid.uuid4())) return self._messages[s]
def _count_messages(self, topic): messages = [] hosts = '%s:%d,%s:%d' % ( self.kafka_brokers[0].host, self.kafka_brokers[0].port, self.kafka_brokers[1].host, self.kafka_brokers[1].port) client = KafkaClient(hosts, clientId="CountMessages", timeout=500) try: yield ensure_topic_creation(client, topic, fully_replicated=False, reactor=self.reactor) # Need to retry this until we have a leader... while True: # Ask the client to load the latest metadata. This may avoid a # NotLeaderForPartitionError I was seeing upon re-start of the # broker. yield client.load_metadata_for_topics(topic) # if there is an error on the metadata for the topic, raise if check_error(client.metadata_error_for_topic(topic), False) is None: break # Ok, should be safe to get the partitions now... partitions = client.topic_partitions[topic] requests = [ FetchRequest(topic, part, 0, 1024 * 1024) for part in partitions ] resps = [] while not resps: try: log.debug("_count_message: Fetching messages") # Prevent log.error() call from causing test failure with patch.object(kclient, 'log'): resps = yield client.send_fetch_request( requests, max_wait_time=400) except (NotLeaderForPartitionError, UnknownTopicOrPartitionError, KafkaUnavailableError): # pragma: no cover log.debug("_count_message: Metadata err, retrying...") yield client.load_metadata_for_topics(topic) except FailedPayloadsError as e: # pragma: no cover if not e.args[1][0][1].check(RequestTimedOutError): raise log.debug("_count_message: Timed out err, retrying...") finally: yield client.close() for fetch_resp in resps: messages.extend(list(fetch_resp.messages)) log.debug("Got %d messages:%r", len(messages), messages) returnValue(len(messages))
def _count_messages(self, topic): log.debug("Counting messages on topic %s", topic) messages = [] client = KafkaClient(self.harness.bootstrap_hosts, clientId="CountMessages", timeout=500, reactor=self.reactor) try: yield ensure_topic_creation(client, topic, fully_replicated=False) # Need to retry this until we have a leader... while True: # Ask the client to load the latest metadata. This may avoid a # NotLeaderForPartitionError I was seeing upon re-start of the # broker. yield client.load_metadata_for_topics(topic) # if there is an error on the metadata for the topic, wait errno = client.metadata_error_for_topic(topic) if errno == 0: break else: log.debug("Topic %s in error errno=%d", topic, errno) yield async_delay(1.0) # Ok, should be safe to get the partitions now... partitions = client.topic_partitions[topic] requests = [ FetchRequest(topic, part, 0, 1024 * 1024) for part in partitions ] resps = [] while not resps: try: log.debug("_count_message: Fetching messages") resps = yield client.send_fetch_request(requests, max_wait_time=400) except (NotLeaderForPartitionError, UnknownTopicOrPartitionError, KafkaUnavailableError): # pragma: no cover log.debug("_count_message: Metadata err, retrying...") yield client.load_metadata_for_topics(topic) except FailedPayloadsError as e: # pragma: no cover if not e.args[1][0][1].check(RequestTimedOutError): raise log.debug("_count_message: Timed out err, retrying...") finally: yield client.close() for fetch_resp in resps: messages.extend(list(fetch_resp.messages)) log.debug("Got %d messages: %r", len(messages), messages) returnValue(len(messages))
def _count_messages(self, topic): messages = [] hosts = '%s:%d,%s:%d' % (self.kafka_brokers[0].host, self.kafka_brokers[0].port, self.kafka_brokers[1].host, self.kafka_brokers[1].port) client = KafkaClient(hosts, clientId="CountMessages", timeout=500) try: yield ensure_topic_creation(client, topic, reactor=self.reactor) # Need to retry this until we have a leader... while True: # Ask the client to load the latest metadata. This may avoid a # NotLeaderForPartitionError I was seeing upon re-start of the # broker. yield client.load_metadata_for_topics(topic) # if there is an error on the metadata for the topic, raise if check_error( client.metadata_error_for_topic(topic), False) is None: break # Ok, should be safe to get the partitions now... partitions = client.topic_partitions[topic] requests = [FetchRequest(topic, part, 0, 1024 * 1024) for part in partitions] resps = [] while not resps: try: log.debug("_count_message: Fetching messages") # Prevent log.error() call from causing test failure with patch.object(kclient, 'log'): resps = yield client.send_fetch_request( requests, max_wait_time=400) except (NotLeaderForPartitionError, UnknownTopicOrPartitionError, KafkaUnavailableError): # pragma: no cover log.debug("_count_message: Metadata err, retrying...") yield client.load_metadata_for_topics(topic) except FailedPayloadsError as e: # pragma: no cover if not e.args[1][0][1].check(RequestTimedOutError): raise log.debug("_count_message: Timed out err, retrying...") finally: yield client.close() for fetch_resp in resps: messages.extend(list(fetch_resp.messages)) log.debug("Got %d messages:%r", len(messages), messages) returnValue(len(messages))
class KafkaIntegrationTestCase(unittest.TestCase): create_client = True topic = None server = None reactor = None def shortDescription(self): """ Show the ID of the test when nose displays its name, rather than a snippet of the docstring. """ return self.id() @deferred(timeout=10) @inlineCallbacks def setUp(self): log.info("Setting up test %s", self.id()) super(KafkaIntegrationTestCase, self).setUp() if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.error('KAFKA_VERSION unset!') return if not self.topic: self.topic = "%s-%s" % (self.id()[self.id().rindex(".") + 1:], random_string(10)) if self.create_client: self.client = KafkaClient('%s:%d' % (self.server.host, self.server.port), clientId=self.topic) yield ensure_topic_creation(self.client, self.topic, fully_replicated=True, reactor=self.reactor) self._messages = {} @deferred(timeout=10) @inlineCallbacks def tearDown(self): log.info("Tearing down test: %r", self) super(KafkaIntegrationTestCase, self).tearDown() if not os.environ.get('KAFKA_VERSION'): # pragma: no cover log.error('KAFKA_VERSION unset!') return if self.create_client: yield self.client.close() # Check for outstanding delayedCalls. Note, this may yield # spurious errors if the class's client has an outstanding # delayed call due to reconnecting. dcs = self.reactor.getDelayedCalls() if dcs: # pragma: no cover log.error("Outstanding Delayed Calls at tearDown: %s\n\n", ' '.join([str(dc) for dc in dcs])) self.assertFalse(dcs) @inlineCallbacks def current_offset(self, topic, partition): offsets, = yield self.client.send_offset_request( [OffsetRequest(topic, partition, -1, 1)]) returnValue(offsets.offsets[0]) def msg(self, s): if s not in self._messages: self._messages[s] = (u'%s-%s-%s' % (s, self.id(), uuid.uuid4())).encode('utf-8') return self._messages[s]