def test_rolling_bounces_will_not_trigger_rebalance_under_static_membership(
self):
self.zookeeper.start()
self.kafka.start()
numThreads = 3
processor1 = StaticMemberTestService(self.test_context, self.kafka,
"consumer-A", numThreads)
processor2 = StaticMemberTestService(self.test_context, self.kafka,
"consumer-B", numThreads)
processor3 = StaticMemberTestService(self.test_context, self.kafka,
"consumer-C", numThreads)
processors = [processor1, processor2, processor3]
self.producer.start()
for processor in processors:
processor.CLEAN_NODE_ENABLED = False
self.set_topics(processor)
verify_running(processor, self.running_message)
self.verify_processing(processors)
# do several rolling bounces
num_bounces = 3
for i in range(0, num_bounces):
for processor in processors:
verify_stopped(processor, self.stopped_message)
verify_running(processor, self.running_message)
stable_generation = -1
for processor in processors:
generations = extract_generation_from_logs(processor)
num_bounce_generations = num_bounces * numThreads
assert num_bounce_generations <= len(generations), \
"Smaller than minimum expected %d generation messages, actual %d" % (num_bounce_generations, len(generations))
for generation in generations[-num_bounce_generations:]:
generation = int(generation)
if stable_generation == -1:
stable_generation = generation
assert stable_generation == generation, \
"Stream rolling bounce have caused unexpected generation bump %d" % generation
self.verify_processing(processors)
stop_processors(processors, self.stopped_message)
self.producer.stop()
self.kafka.stop()
self.zookeeper.stop()
def maybe_upgrade_rolling_bounce_and_verify(self,
processors,
previous_phase,
current_phase,
upgrade_from_version=None):
for processor in processors:
# stop the processor in prep for setting "update.from" or removing "update.from"
verify_stopped(processor, previous_phase + self.stopped_message)
# upgrade to version with cooperative rebalance
processor.set_version("")
processor.set_upgrade_phase(current_phase)
if upgrade_from_version is not None:
# need to remove minor version numbers for check of valid upgrade from numbers
upgrade_version = upgrade_from_version[:upgrade_from_version.
rfind('.')]
rebalance_mode_msg = self.cooperative_turned_off_msg % upgrade_version
else:
upgrade_version = None
rebalance_mode_msg = self.cooperative_enabled_msg
self.set_props(processor, upgrade_version)
node = processor.node
with node.account.monitor_log(
processor.STDOUT_FILE) as stdout_monitor:
with node.account.monitor_log(
processor.LOG_FILE) as log_monitor:
processor.start()
# verify correct rebalance mode either turned off for upgrade or enabled after upgrade
log_monitor.wait_until(
rebalance_mode_msg,
timeout_sec=60,
err_msg="Never saw '%s' message " % rebalance_mode_msg
+ str(processor.node.account))
# verify rebalanced into a running state
rebalance_msg = current_phase + self.running_state_msg
stdout_monitor.wait_until(
rebalance_msg,
timeout_sec=60,
err_msg="Never saw '%s' message " % rebalance_msg +
str(processor.node.account))
# verify processing
verify_processing_msg = current_phase + self.processing_message
stdout_monitor.wait_until(
verify_processing_msg,
timeout_sec=60,
err_msg="Never saw '%s' message " % verify_processing_msg +
str(processor.node.account))
def test_upgrade_topology_with_named_repartition_topic(self):
self.zookeeper.start()
self.kafka.start()
processor1 = StreamsNamedRepartitionTopicService(
self.test_context, self.kafka)
processor2 = StreamsNamedRepartitionTopicService(
self.test_context, self.kafka)
processor3 = StreamsNamedRepartitionTopicService(
self.test_context, self.kafka)
processors = [processor1, processor2, processor3]
self.producer.start()
for processor in processors:
processor.CLEAN_NODE_ENABLED = False
self.set_topics(processor)
verify_running(processor, 'REBALANCING -> RUNNING')
self.verify_processing(processors)
# do rolling upgrade
for processor in processors:
verify_stopped(processor, self.stopped_message)
# will tell app to add operations before repartition topic
processor.ADD_ADDITIONAL_OPS = 'true'
verify_running(processor, 'UPDATED Topology')
self.verify_processing(processors)
stop_processors(processors, self.stopped_message)
self.producer.stop()
self.kafka.stop()
self.zookeeper.stop()
