def test_rolling_bounces_will_not_trigger_rebalance_under_static_membership( self): self.zookeeper.start() self.kafka.start() numThreads = 3 processor1 = StaticMemberTestService(self.test_context, self.kafka, "consumer-A", numThreads) processor2 = StaticMemberTestService(self.test_context, self.kafka, "consumer-B", numThreads) processor3 = StaticMemberTestService(self.test_context, self.kafka, "consumer-C", numThreads) processors = [processor1, processor2, processor3] self.producer.start() for processor in processors: processor.CLEAN_NODE_ENABLED = False self.set_topics(processor) verify_running(processor, self.running_message) self.verify_processing(processors) # do several rolling bounces num_bounces = 3 for i in range(0, num_bounces): for processor in processors: verify_stopped(processor, self.stopped_message) verify_running(processor, self.running_message) stable_generation = -1 for processor in processors: generations = extract_generation_from_logs(processor) num_bounce_generations = num_bounces * numThreads assert num_bounce_generations <= len(generations), \ "Smaller than minimum expected %d generation messages, actual %d" % (num_bounce_generations, len(generations)) for generation in generations[-num_bounce_generations:]: generation = int(generation) if stable_generation == -1: stable_generation = generation assert stable_generation == generation, \ "Stream rolling bounce have caused unexpected generation bump %d" % generation self.verify_processing(processors) stop_processors(processors, self.stopped_message) self.producer.stop() self.kafka.stop() self.zookeeper.stop()
def maybe_upgrade_rolling_bounce_and_verify(self, processors, previous_phase, current_phase, upgrade_from_version=None): for processor in processors: # stop the processor in prep for setting "update.from" or removing "update.from" verify_stopped(processor, previous_phase + self.stopped_message) # upgrade to version with cooperative rebalance processor.set_version("") processor.set_upgrade_phase(current_phase) if upgrade_from_version is not None: # need to remove minor version numbers for check of valid upgrade from numbers upgrade_version = upgrade_from_version[:upgrade_from_version. rfind('.')] rebalance_mode_msg = self.cooperative_turned_off_msg % upgrade_version else: upgrade_version = None rebalance_mode_msg = self.cooperative_enabled_msg self.set_props(processor, upgrade_version) node = processor.node with node.account.monitor_log( processor.STDOUT_FILE) as stdout_monitor: with node.account.monitor_log( processor.LOG_FILE) as log_monitor: processor.start() # verify correct rebalance mode either turned off for upgrade or enabled after upgrade log_monitor.wait_until( rebalance_mode_msg, timeout_sec=60, err_msg="Never saw '%s' message " % rebalance_mode_msg + str(processor.node.account)) # verify rebalanced into a running state rebalance_msg = current_phase + self.running_state_msg stdout_monitor.wait_until( rebalance_msg, timeout_sec=60, err_msg="Never saw '%s' message " % rebalance_msg + str(processor.node.account)) # verify processing verify_processing_msg = current_phase + self.processing_message stdout_monitor.wait_until( verify_processing_msg, timeout_sec=60, err_msg="Never saw '%s' message " % verify_processing_msg + str(processor.node.account))
def test_upgrade_topology_with_named_repartition_topic(self): self.zookeeper.start() self.kafka.start() processor1 = StreamsNamedRepartitionTopicService( self.test_context, self.kafka) processor2 = StreamsNamedRepartitionTopicService( self.test_context, self.kafka) processor3 = StreamsNamedRepartitionTopicService( self.test_context, self.kafka) processors = [processor1, processor2, processor3] self.producer.start() for processor in processors: processor.CLEAN_NODE_ENABLED = False self.set_topics(processor) verify_running(processor, 'REBALANCING -> RUNNING') self.verify_processing(processors) # do rolling upgrade for processor in processors: verify_stopped(processor, self.stopped_message) # will tell app to add operations before repartition topic processor.ADD_ADDITIONAL_OPS = 'true' verify_running(processor, 'UPDATED Topology') self.verify_processing(processors) stop_processors(processors, self.stopped_message) self.producer.stop() self.kafka.stop() self.zookeeper.stop()