def test_fail_fast_on_incompatible_brokers(self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, False) processor.start() processor.node.account.ssh(processor.start_cmd(processor.node)) with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor: monitor.wait_until('Exception in thread "main" org.apache.kafka.streams.errors.StreamsException: Kafka Streams requires broker version 0.10.1.x or higher.', timeout_sec=60, err_msg="Never saw 'Streams requires broker verion 0.10.1+' error message " + str(processor.node.account)) self.kafka.stop()
def test_fail_fast_on_incompatible_brokers_if_eos_enabled(self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, True) processor.start() processor.node.account.ssh(processor.start_cmd(processor.node)) with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor: monitor.wait_until('Exception in thread "main" org.apache.kafka.streams.errors.StreamsException: Setting processing.guarantee=exactly_once requires broker version 0.11.0.x or higher.', timeout_sec=60, err_msg="Never saw 'EOS requires broker version 0.11+' error message " + str(processor.node.account)) self.kafka.stop()
def test_timeout_on_pre_010_brokers(self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, False) processor.start() processor.node.account.ssh(processor.start_cmd(processor.node)) with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor: monitor.wait_until('Exception in thread "main" org.apache.kafka.streams.errors.BrokerNotFoundException: Could not find any available broker.', timeout_sec=60, err_msg="Never saw 'no available brokers' error message " + str(processor.node.account)) self.kafka.stop()
def test_fail_fast_on_incompatible_brokers(self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, False) processor.start() processor.node.account.ssh(processor.start_cmd(processor.node)) with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor: monitor.wait_until('FATAL: An unexpected exception org.apache.kafka.streams.errors.StreamsException: Could not create topic kafka-streams-system-test-broker-compatibility-KSTREAM-AGGREGATE-STATE-STORE-0000000001-changelog.', timeout_sec=60, err_msg="Never saw 'FATAL: An unexpected exception org.apache.kafka.streams.errors.StreamsException: Could not create topic kafka-streams-system-test-broker-compatibility-KSTREAM-AGGREGATE-STATE-STORE-0000000001-changelog.' error message " + str(processor.node.account)) self.kafka.stop()
def test_fail_fast_on_incompatible_brokers_if_eos_enabled(self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka, True) processor.start() processor.node.account.ssh(processor.start_cmd(processor.node)) with processor.node.account.monitor_log(processor.STDERR_FILE) as monitor: monitor.wait_until('FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support LIST_OFFSETS ', timeout_sec=60, err_msg="Never saw 'FATAL: An unexpected exception org.apache.kafka.common.errors.UnsupportedVersionException: The broker does not support LIST_OFFSETS ' error message " + str(processor.node.account)) self.kafka.stop()
class StreamsBrokerCompatibility(Test): """ These tests validate that Streams v0.10.2+ can connect to older brokers v0.10.1+ and that Streams fails fast for pre-0.10.0 brokers """ input = "brokerCompatibilitySourceTopic" output = "brokerCompatibilitySinkTopic" def __init__(self, test_context): super(StreamsBrokerCompatibility, self).__init__(test_context=test_context) self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(test_context, num_nodes=1, zk=self.zk, topics={ self.input: { 'partitions': 1, 'replication-factor': 1 }, self.output: { 'partitions': 1, 'replication-factor': 1 } }) self.processor = StreamsBrokerCompatibilityService( self.test_context, self.kafka) self.consumer = VerifiableConsumer( test_context, 1, self.kafka, self.output, "stream-broker-compatibility-verify-consumer") def setUp(self): self.zk.start() @parametrize(broker_version=str(DEV_BRANCH)) @parametrize(broker_version=str(LATEST_0_10_1)) def test_compatible_brokers(self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() self.processor.start() self.consumer.start() self.processor.wait() wait_until( lambda: self.consumer.total_consumed() > 0, timeout_sec=30, err_msg= "Did expect to read a message but got none within 30 seconds.") self.consumer.stop() self.kafka.stop() @parametrize(broker_version=str(LATEST_0_10_0)) def test_fail_fast_on_incompatible_brokers(self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() self.processor.start() self.processor.node.account.ssh( self.processor.start_cmd(self.processor.node)) with self.processor.node.account.monitor_log( self.processor.STDERR_FILE) as monitor: monitor.wait_until( 'Exception in thread "main" org.apache.kafka.streams.errors.StreamsException: Kafka Streams requires broker version 0.10.1.x or higher.', timeout_sec=60, err_msg="Never saw 'incompatible broker' error message " + str(self.processor.node.account)) self.kafka.stop()
class StreamsBrokerCompatibility(Test): """ These tests validate that Streams v0.10.2+ can connect to older brokers v0.10+ and that Streams fails fast for pre-0.10 brokers """ input = "brokerCompatibilitySourceTopic" output = "brokerCompatibilitySinkTopic" def __init__(self, test_context): super(StreamsBrokerCompatibility, self).__init__(test_context=test_context) self.zk = ZookeeperService(test_context, num_nodes=1) self.kafka = KafkaService(test_context, num_nodes=1, zk=self.zk, topics={ self.input: {'partitions': 1, 'replication-factor': 1}, self.output: {'partitions': 1, 'replication-factor': 1} }) self.processor = StreamsBrokerCompatibilityService(self.test_context, self.kafka) self.consumer = VerifiableConsumer(test_context, 1, self.kafka, self.output, "stream-broker-compatibility-verify-consumer") def setUp(self): self.zk.start() @parametrize(broker_version=str(DEV_BRANCH)) @parametrize(broker_version=str(LATEST_0_10_1)) def test_compatible_brokers(self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() self.processor.start() self.consumer.start() self.processor.wait() num_consumed_mgs = self.consumer.total_consumed() self.consumer.stop() self.kafka.stop() assert num_consumed_mgs == 1, \ "Did expect to read exactly one message but got %d" % num_consumed_mgs @parametrize(broker_version=str(LATEST_0_10_0)) def test_fail_fast_on_incompatible_brokers(self, broker_version): self.kafka.set_version(KafkaVersion(broker_version)) self.kafka.start() self.processor.start() self.processor.node.account.ssh(self.processor.start_cmd(self.processor.node)) with self.processor.node.account.monitor_log(self.processor.STDERR_FILE) as monitor: monitor.wait_until('Exception in thread "main" org.apache.kafka.streams.errors.StreamsException: Kafka Streams requires broker version 0.10.1.x or higher.', timeout_sec=60, err_msg="Never saw 'incompatible broker' error message " + str(self.processor.node.account)) self.kafka.stop()