def test_log_fixup_after_snapshot_delivery(cluster): """ Log must be restarted when loading a snapshot. """ cluster.create(3) cluster.node(1).raft_exec('INCR', 'key') cluster.node(1).raft_exec('INCR', 'key') cluster.node(2).terminate() cluster.node(1).raft_exec('INCR', 'key') assert cluster.node(1).client.execute_command( 'RAFT.DEBUG', 'COMPACT') == b'OK' # Confirm node 2's log starts from 1 log = RaftLog(cluster.node(2).raftlog) log.read() assert log.header().snapshot_index() == 0 cluster.node(2).start() cluster.node(2).wait_for_info_param('state', 'up') # node 2 must get a snapshot to sync, make sure this happens and that # the log is ok. cluster.node(2).wait_for_current_index(8) log = RaftLog(cluster.node(2).raftlog) log.read() assert log.header().snapshot_index() == 8
def test_uncommitted_log_rewrite(cluster): cluster.create(3) # Log contains 5 config entries # Take down majority to create uncommitted entries and check rewrite cluster.node(1).raft_exec('SET', 'key', 'value') # Entry idx 6 cluster.node(2).terminate() cluster.node(3).terminate() conn = cluster.node(1).client.connection_pool.get_connection('RAFT') conn.send_command('RAFT', 'SET', 'key2', 'value2') # Entry idx 7 assert cluster.node(1).current_index() == 7 assert cluster.node(1).commit_index() == 6 assert cluster.node(1).client.execute_command( 'RAFT.DEBUG', 'COMPACT') == b'OK' assert cluster.node(1).raft_info()['log_entries'] == 1 log = RaftLog(cluster.node(1).raftlog) log.read() assert log.entry_count(LogEntry.LogType.NORMAL) == 1 cluster.node(1).kill() cluster.node(1).start() cluster.node(1).wait_for_info_param('state', 'up') assert cluster.node(1).current_index() == 7 assert cluster.node(1).commit_index() == 6 assert cluster.node(1).raft_info()['log_entries'] == 1
def test_log_rollback(cluster): """ Rollback of log entries that were written in the minority. """ cluster.create(3) assert cluster.leader == 1 assert cluster.raft_exec('INCRBY', 'key', '111') == 111 # Break cluster cluster.node(2).terminate() cluster.node(3).terminate() # Load a command which can't be committed assert cluster.node(1).current_index() == 6 conn = cluster.node(1).client.connection_pool.get_connection('RAFT') conn.send_command('RAFT', 'INCRBY', 'key', '222') assert cluster.node(1).current_index() == 7 cluster.node(1).terminate() # We want to be sure the last entry is in the log log = RaftLog(cluster.node(1).raftlog) log.read() assert log.entry_count() == 7 # Restart the cluster without node 1, make sure the write was # not committed. cluster.node(2).start() cluster.node(3).start() cluster.node(2).wait_for_election() assert cluster.node(2).current_index() == 7 # 6 + 1 no-op entry # Restart node 1 cluster.node(1).start() cluster.node(1).wait_for_election() # Make another write and make sure it overwrites the previous one in # node 1's log assert cluster.raft_exec('INCRBY', 'key', '333') == 444 cluster.wait_for_unanimity() # Make sure log reflects the change log.reset() log.read() assert match(r'.*INCRBY.*333', str(log.entries[-1].data()))
def test_new_uncommitted_during_rewrite(cluster): cluster.create(3) # Take down majority to create uncommitted entries and check rewrite cluster.node(1).raft_exec('SET', 'key', '1') # Initiate compaction and wait to see it's in progress conn = cluster.node(1).client.connection_pool.get_connection('COMPACT') conn.send_command('RAFT.DEBUG', 'COMPACT', '2') cluster.node(1).wait_for_info_param('snapshot_in_progress', 'yes') assert cluster.node(1).raft_info()['snapshot_in_progress'] == 'yes' # Send a bunch of writes cluster.node(1).raft_exec('INCRBY', 'key', '2') cluster.node(1).raft_exec('INCRBY', 'key', '3') cluster.node(1).raft_exec('INCRBY', 'key', '4') # Wait for compaction to end assert cluster.node(1).raft_info()['snapshot_in_progress'] == 'yes' cluster.node(1).wait_for_info_param('snapshot_in_progress', 'no') # Make sure our writes made it to the log log = RaftLog(cluster.node(1).raftlog) log.read() assert log.entry_count(LogEntry.LogType.NORMAL) == 3 # Extra check -- Make sure we can read it back. Note that we need to start # all nodes because we don't log the commit index. cluster.node(1).terminate() cluster.node(2).terminate() cluster.node(3).terminate() cluster.node(1).start() cluster.node(2).start() cluster.node(3).start() cluster.node(1).wait_for_info_param('state', 'up') # Make sure cluster state is as expected assert cluster.raft_exec('get', 'key') == b'10' # Make sure node 1 state is as expected cluster.node(1).wait_for_log_applied() assert cluster.node(1).client.get('key') == b'10'
def test_all_committed_log_rewrite(cluster): """ Log rewrite operation when all entries are committed, so we expect an empty log. """ cluster.create(3) cluster.node(1).raft_exec('SET', 'key1', 'value') cluster.node(1).raft_exec('SET', 'key2', 'value') cluster.node(1).raft_exec('SET', 'key3', 'value') cluster.wait_for_unanimity() assert cluster.node(1).client.execute_command( 'RAFT.DEBUG', 'COMPACT') == b'OK' assert cluster.node(1).raft_info()['log_entries'] == 0 # Make sure we have no log entries! log = RaftLog(cluster.node(1).raftlog) log.read() assert log.entry_count(LogEntry.LogType.NORMAL) == 0