def test_3x5_zookeeper_invalid_offset(): """Test recovery from bad ZK offset value If ZooKeeper stored an invalid start offset, we should print an ERROR and start from the latest.""" log_break("test_3x5_zookeeper_invalid_offset") c1 = ZKConsumer(ZK_CONNECT_STR, "group_3x5_zookeeper_invalid_offset", "topic_3x5_zookeeper_invalid_offset", autocommit=True) send_to_all_partitions("topic_3x5_zookeeper_invalid_offset", ["hello"]) time.sleep(MESSAGE_DELAY_SECS) # The following fetch will also save the ZK offset (autocommit=True) result = c1.fetch() # Now let's reach into ZooKeeper and manually set the offset to something # out of range. z1 = c1._zk_util bps_to_fake_offsets = dict((bp, 1000) for bp in c1.broker_partitions) z1.save_offsets_for(c1.consumer_group, bps_to_fake_offsets) c1.close() time.sleep(MESSAGE_DELAY_SECS) # Now delete c1, and create c2, which will take over all of it's partitions c2 = ZKConsumer(ZK_CONNECT_STR, "group_3x5_zookeeper_invalid_offset", "topic_3x5_zookeeper_invalid_offset", autocommit=True) # This should detect that the values in ZK are bad, and put us at the real # end offset. c2.fetch() send_to_all_partitions("topic_3x5_zookeeper_invalid_offset", ["world"]) time.sleep(MESSAGE_DELAY_SECS) result = c2.fetch() assert result for msg_set in result: assert_equals(msg_set.messages, ["world"])
def test_3x5_zookeeper_invalid_offset(): """Test recovery from bad ZK offset value If ZooKeeper stored an invalid start offset, we should print an ERROR and start from the latest.""" log_break("test_3x5_zookeeper_invalid_offset") c1 = ZKConsumer(ZK_CONNECT_STR, "group_3x5_zookeeper_invalid_offset", "topic_3x5_zookeeper_invalid_offset", autocommit=True) send_to_all_partitions(5, "topic_3x5_zookeeper_invalid_offset", ["hello"]) delay() # The following fetch will also save the ZK offset (autocommit=True) result = c1.fetch() # Now let's reach into ZooKeeper and manually set the offset to something # out of range. z1 = c1._zk_util bps_to_fake_offsets = dict((bp, 1000) for bp in c1.broker_partitions) z1.save_offsets_for(c1.consumer_group, bps_to_fake_offsets) c1.close() delay() # Now delete c1, and create c2, which will take over all of it's partitions c2 = ZKConsumer(ZK_CONNECT_STR, "group_3x5_zookeeper_invalid_offset", "topic_3x5_zookeeper_invalid_offset", autocommit=True) # This should detect that the values in ZK are bad, and put us at the real # end offset. c2.fetch() send_to_all_partitions(5, "topic_3x5_zookeeper_invalid_offset", ["world"]) delay() result = c2.fetch() assert result for msg_set in result: assert_equals(msg_set.messages, ["world"])