def test_3x5_zookeeper_invalid_offset():
"""Test recovery from bad ZK offset value
If ZooKeeper stored an invalid start offset, we should print an ERROR
and start from the latest."""
log_break("test_3x5_zookeeper_invalid_offset")
c1 = ZKConsumer(ZK_CONNECT_STR,
"group_3x5_zookeeper_invalid_offset",
"topic_3x5_zookeeper_invalid_offset",
autocommit=True)
send_to_all_partitions("topic_3x5_zookeeper_invalid_offset", ["hello"])
time.sleep(MESSAGE_DELAY_SECS)
# The following fetch will also save the ZK offset (autocommit=True)
result = c1.fetch()
# Now let's reach into ZooKeeper and manually set the offset to something
# out of range.
z1 = c1._zk_util
bps_to_fake_offsets = dict((bp, 1000) for bp in c1.broker_partitions)
z1.save_offsets_for(c1.consumer_group, bps_to_fake_offsets)
c1.close()
time.sleep(MESSAGE_DELAY_SECS)
# Now delete c1, and create c2, which will take over all of it's partitions
c2 = ZKConsumer(ZK_CONNECT_STR,
"group_3x5_zookeeper_invalid_offset",
"topic_3x5_zookeeper_invalid_offset",
autocommit=True)
# This should detect that the values in ZK are bad, and put us at the real
# end offset.
c2.fetch()
send_to_all_partitions("topic_3x5_zookeeper_invalid_offset", ["world"])
time.sleep(MESSAGE_DELAY_SECS)
result = c2.fetch()
assert result
for msg_set in result:
assert_equals(msg_set.messages, ["world"])
def test_3x5_zookeeper_invalid_offset():
"""Test recovery from bad ZK offset value
If ZooKeeper stored an invalid start offset, we should print an ERROR
and start from the latest."""
log_break("test_3x5_zookeeper_invalid_offset")
c1 = ZKConsumer(ZK_CONNECT_STR,
"group_3x5_zookeeper_invalid_offset",
"topic_3x5_zookeeper_invalid_offset",
autocommit=True)
send_to_all_partitions(5, "topic_3x5_zookeeper_invalid_offset", ["hello"])
delay()
# The following fetch will also save the ZK offset (autocommit=True)
result = c1.fetch()
# Now let's reach into ZooKeeper and manually set the offset to something
# out of range.
z1 = c1._zk_util
bps_to_fake_offsets = dict((bp, 1000) for bp in c1.broker_partitions)
z1.save_offsets_for(c1.consumer_group, bps_to_fake_offsets)
c1.close()
delay()
# Now delete c1, and create c2, which will take over all of it's partitions
c2 = ZKConsumer(ZK_CONNECT_STR,
"group_3x5_zookeeper_invalid_offset",
"topic_3x5_zookeeper_invalid_offset",
autocommit=True)
# This should detect that the values in ZK are bad, and put us at the real
# end offset.
c2.fetch()
send_to_all_partitions(5, "topic_3x5_zookeeper_invalid_offset", ["world"])
delay()
result = c2.fetch()
assert result
for msg_set in result:
assert_equals(msg_set.messages, ["world"])
