def test_log_fixup_after_snapshot_delivery(cluster):
"""
Log must be restarted when loading a snapshot.
"""
cluster.create(3)
cluster.node(1).raft_exec('INCR', 'key')
cluster.node(1).raft_exec('INCR', 'key')
cluster.node(2).terminate()
cluster.node(1).raft_exec('INCR', 'key')
assert cluster.node(1).client.execute_command(
'RAFT.DEBUG', 'COMPACT') == b'OK'
# Confirm node 2's log starts from 1
log = RaftLog(cluster.node(2).raftlog)
log.read()
assert log.header().snapshot_index() == 0
cluster.node(2).start()
cluster.node(2).wait_for_info_param('state', 'up')
# node 2 must get a snapshot to sync, make sure this happens and that
# the log is ok.
cluster.node(2).wait_for_current_index(8)
log = RaftLog(cluster.node(2).raftlog)
log.read()
assert log.header().snapshot_index() == 8
def test_uncommitted_log_rewrite(cluster):
cluster.create(3)
# Log contains 5 config entries
# Take down majority to create uncommitted entries and check rewrite
cluster.node(1).raft_exec('SET', 'key', 'value') # Entry idx 6
cluster.node(2).terminate()
cluster.node(3).terminate()
conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
conn.send_command('RAFT', 'SET', 'key2', 'value2') # Entry idx 7
assert cluster.node(1).current_index() == 7
assert cluster.node(1).commit_index() == 6
assert cluster.node(1).client.execute_command(
'RAFT.DEBUG', 'COMPACT') == b'OK'
assert cluster.node(1).raft_info()['log_entries'] == 1
log = RaftLog(cluster.node(1).raftlog)
log.read()
assert log.entry_count(LogEntry.LogType.NORMAL) == 1
cluster.node(1).kill()
cluster.node(1).start()
cluster.node(1).wait_for_info_param('state', 'up')
assert cluster.node(1).current_index() == 7
assert cluster.node(1).commit_index() == 6
assert cluster.node(1).raft_info()['log_entries'] == 1
def test_log_rollback(cluster):
"""
Rollback of log entries that were written in the minority.
"""
cluster.create(3)
assert cluster.leader == 1
assert cluster.raft_exec('INCRBY', 'key', '111') == 111
# Break cluster
cluster.node(2).terminate()
cluster.node(3).terminate()
# Load a command which can't be committed
assert cluster.node(1).current_index() == 6
conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
conn.send_command('RAFT', 'INCRBY', 'key', '222')
assert cluster.node(1).current_index() == 7
cluster.node(1).terminate()
# We want to be sure the last entry is in the log
log = RaftLog(cluster.node(1).raftlog)
log.read()
assert log.entry_count() == 7
# Restart the cluster without node 1, make sure the write was
# not committed.
cluster.node(2).start()
cluster.node(3).start()
cluster.node(2).wait_for_election()
assert cluster.node(2).current_index() == 7 # 6 + 1 no-op entry
# Restart node 1
cluster.node(1).start()
cluster.node(1).wait_for_election()
# Make another write and make sure it overwrites the previous one in
# node 1's log
assert cluster.raft_exec('INCRBY', 'key', '333') == 444
cluster.wait_for_unanimity()
# Make sure log reflects the change
log.reset()
log.read()
assert match(r'.*INCRBY.*333', str(log.entries[-1].data()))
def test_new_uncommitted_during_rewrite(cluster):
cluster.create(3)
# Take down majority to create uncommitted entries and check rewrite
cluster.node(1).raft_exec('SET', 'key', '1')
# Initiate compaction and wait to see it's in progress
conn = cluster.node(1).client.connection_pool.get_connection('COMPACT')
conn.send_command('RAFT.DEBUG', 'COMPACT', '2')
cluster.node(1).wait_for_info_param('snapshot_in_progress', 'yes')
assert cluster.node(1).raft_info()['snapshot_in_progress'] == 'yes'
# Send a bunch of writes
cluster.node(1).raft_exec('INCRBY', 'key', '2')
cluster.node(1).raft_exec('INCRBY', 'key', '3')
cluster.node(1).raft_exec('INCRBY', 'key', '4')
# Wait for compaction to end
assert cluster.node(1).raft_info()['snapshot_in_progress'] == 'yes'
cluster.node(1).wait_for_info_param('snapshot_in_progress', 'no')
# Make sure our writes made it to the log
log = RaftLog(cluster.node(1).raftlog)
log.read()
assert log.entry_count(LogEntry.LogType.NORMAL) == 3
# Extra check -- Make sure we can read it back. Note that we need to start
# all nodes because we don't log the commit index.
cluster.node(1).terminate()
cluster.node(2).terminate()
cluster.node(3).terminate()
cluster.node(1).start()
cluster.node(2).start()
cluster.node(3).start()
cluster.node(1).wait_for_info_param('state', 'up')
# Make sure cluster state is as expected
assert cluster.raft_exec('get', 'key') == b'10'
# Make sure node 1 state is as expected
cluster.node(1).wait_for_log_applied()
assert cluster.node(1).client.get('key') == b'10'
def test_all_committed_log_rewrite(cluster):
"""
Log rewrite operation when all entries are committed, so we expect an
empty log.
"""
cluster.create(3)
cluster.node(1).raft_exec('SET', 'key1', 'value')
cluster.node(1).raft_exec('SET', 'key2', 'value')
cluster.node(1).raft_exec('SET', 'key3', 'value')
cluster.wait_for_unanimity()
assert cluster.node(1).client.execute_command(
'RAFT.DEBUG', 'COMPACT') == b'OK'
assert cluster.node(1).raft_info()['log_entries'] == 0
# Make sure we have no log entries!
log = RaftLog(cluster.node(1).raftlog)
log.read()
assert log.entry_count(LogEntry.LogType.NORMAL) == 0