def test_recover_raftlog(tmp_path):
for i in range(3):
with open(tmp_path / str(i), "w") as f:
f.write(LogEntry(i, i).toyaml())
l1 = RaftLog(tmp_path)
assert l1.measure_log() == (2, 3)
assert l1 == [LogEntry(0, 0), LogEntry(1, 1), LogEntry(2, 2)]
def test_log_fixup_after_snapshot_delivery(cluster):
"""
Log must be restarted when loading a snapshot.
"""
cluster.create(3)
cluster.node(1).raft_exec('INCR', 'key')
cluster.node(1).raft_exec('INCR', 'key')
cluster.node(2).terminate()
cluster.node(1).raft_exec('INCR', 'key')
assert cluster.node(1).client.execute_command(
'RAFT.DEBUG', 'COMPACT') == b'OK'
# Confirm node 2's log starts from 1
log = RaftLog(cluster.node(2).raftlog)
log.read()
assert log.header().snapshot_index() == 0
cluster.node(2).start()
cluster.node(2).wait_for_info_param('state', 'up')
# node 2 must get a snapshot to sync, make sure this happens and that
# the log is ok.
cluster.node(2).wait_for_current_index(8)
log = RaftLog(cluster.node(2).raftlog)
log.read()
assert log.header().snapshot_index() == 8
def test_uncommitted_log_rewrite(cluster):
cluster.create(3)
# Log contains 5 config entries
# Take down majority to create uncommitted entries and check rewrite
cluster.node(1).raft_exec('SET', 'key', 'value') # Entry idx 6
cluster.node(2).terminate()
cluster.node(3).terminate()
conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
conn.send_command('RAFT', 'SET', 'key2', 'value2') # Entry idx 7
assert cluster.node(1).current_index() == 7
assert cluster.node(1).commit_index() == 6
assert cluster.node(1).client.execute_command(
'RAFT.DEBUG', 'COMPACT') == b'OK'
assert cluster.node(1).raft_info()['log_entries'] == 1
log = RaftLog(cluster.node(1).raftlog)
log.read()
assert log.entry_count(LogEntry.LogType.NORMAL) == 1
cluster.node(1).kill()
cluster.node(1).start()
cluster.node(1).wait_for_info_param('state', 'up')
assert cluster.node(1).current_index() == 7
assert cluster.node(1).commit_index() == 6
assert cluster.node(1).raft_info()['log_entries'] == 1
def test_recover_raftlog_2(tmp_path):
l1 = RaftLog(tmp_path)
l1.append_entries(
0, None,
[LogEntry(1, 1), LogEntry(2, 2),
LogEntry(3, 3)])
l2 = RaftLog(tmp_path)
assert l1 == l2
def test_recover_raftlog_inconsistent_1(tmp_path):
# file with index 0 is missing
for i in range(3):
with open(tmp_path / str(i + 1), "w") as f:
f.write(LogEntry(i, i).toyaml())
with pytest.raises(RaftStoredLogInconsistent):
RaftLog(tmp_path)
def __init__(self, id, config, queue_maxsize=10):
self.logger.info(
f"Setting up raft server with id {id}, configuration {config}.")
directory = pathlib.Path(
config[id]["dir"]) if config[id]["dir"] else None
# persistent (log is internally persisted, so just need to persist the others)
# Note: the log assumes all integer named files are log entries
if directory and not directory.is_dir():
directory.mkdir()
self.config_file = directory / "server-config.yaml" if directory else None
if self.config_file and self.config_file.is_file():
with open(self.config_file) as f:
config_from_file = yaml.load(f)
if config_from_file["config"] != config or config_from_file[
"id"] != id:
raise RaftServerInconsistentConfigError(
config_from_file, id, config)
else:
config_from_file = {}
self.id = config_from_file.get("id", id)
self.config = config_from_file.get("config", config)
self.currentTerm = config_from_file.get("currentTerm", 0)
self.votedFor = config_from_file.get("votedFor", None)
self.persist()
self.log = (RaftLog(directory) if directory else RaftLog()
) # persists itself; but by above dir has been created
# volatile
self.commitIndex = 0
self.lastApplied = -1
self.state = Follower(self)
self.leaderId = None
self.clientsockets = {i: None for i, _ in self.config.items()}
self.sendQueue = Queue(queue_maxsize)
self.recvQueue = Queue(queue_maxsize)
self.isRunning = True
self.lock = threading.Lock()
self.server_str = f"RaftServer({self.id})"
self.slow_step = 0
self.machine = StateMachine()
def test_recover_raftlog_inconsistent_2(tmp_path):
# file with index 1 has invalid digest
for i in range(3):
with open(tmp_path / str(i), "w") as f:
if i == 1:
f.write(LogEntry(i, i).toyaml()[:-3])
else:
f.write(LogEntry(i, i).toyaml())
with pytest.raises(RaftStoredLogInconsistent):
RaftLog(tmp_path)
def test_new_uncommitted_during_rewrite(cluster):
cluster.create(3)
# Take down majority to create uncommitted entries and check rewrite
cluster.node(1).raft_exec('SET', 'key', '1')
# Initiate compaction and wait to see it's in progress
conn = cluster.node(1).client.connection_pool.get_connection('COMPACT')
conn.send_command('RAFT.DEBUG', 'COMPACT', '2')
cluster.node(1).wait_for_info_param('snapshot_in_progress', 'yes')
assert cluster.node(1).raft_info()['snapshot_in_progress'] == 'yes'
# Send a bunch of writes
cluster.node(1).raft_exec('INCRBY', 'key', '2')
cluster.node(1).raft_exec('INCRBY', 'key', '3')
cluster.node(1).raft_exec('INCRBY', 'key', '4')
# Wait for compaction to end
assert cluster.node(1).raft_info()['snapshot_in_progress'] == 'yes'
cluster.node(1).wait_for_info_param('snapshot_in_progress', 'no')
# Make sure our writes made it to the log
log = RaftLog(cluster.node(1).raftlog)
log.read()
assert log.entry_count(LogEntry.LogType.NORMAL) == 3
# Extra check -- Make sure we can read it back. Note that we need to start
# all nodes because we don't log the commit index.
cluster.node(1).terminate()
cluster.node(2).terminate()
cluster.node(3).terminate()
cluster.node(1).start()
cluster.node(2).start()
cluster.node(3).start()
cluster.node(1).wait_for_info_param('state', 'up')
# Make sure cluster state is as expected
assert cluster.raft_exec('get', 'key') == b'10'
# Make sure node 1 state is as expected
cluster.node(1).wait_for_log_applied()
assert cluster.node(1).client.get('key') == b'10'
def test_all_committed_log_rewrite(cluster):
"""
Log rewrite operation when all entries are committed, so we expect an
empty log.
"""
cluster.create(3)
cluster.node(1).raft_exec('SET', 'key1', 'value')
cluster.node(1).raft_exec('SET', 'key2', 'value')
cluster.node(1).raft_exec('SET', 'key3', 'value')
cluster.wait_for_unanimity()
assert cluster.node(1).client.execute_command(
'RAFT.DEBUG', 'COMPACT') == b'OK'
assert cluster.node(1).raft_info()['log_entries'] == 0
# Make sure we have no log entries!
log = RaftLog(cluster.node(1).raftlog)
log.read()
assert log.entry_count(LogEntry.LogType.NORMAL) == 0
def test_log_rollback(cluster):
"""
Rollback of log entries that were written in the minority.
"""
cluster.create(3)
assert cluster.leader == 1
assert cluster.raft_exec('INCRBY', 'key', '111') == 111
# Break cluster
cluster.node(2).terminate()
cluster.node(3).terminate()
# Load a command which can't be committed
assert cluster.node(1).current_index() == 6
conn = cluster.node(1).client.connection_pool.get_connection('RAFT')
conn.send_command('RAFT', 'INCRBY', 'key', '222')
assert cluster.node(1).current_index() == 7
cluster.node(1).terminate()
# We want to be sure the last entry is in the log
log = RaftLog(cluster.node(1).raftlog)
log.read()
assert log.entry_count() == 7
# Restart the cluster without node 1, make sure the write was
# not committed.
cluster.node(2).start()
cluster.node(3).start()
cluster.node(2).wait_for_election()
assert cluster.node(2).current_index() == 7 # 6 + 1 no-op entry
# Restart node 1
cluster.node(1).start()
cluster.node(1).wait_for_election()
# Make another write and make sure it overwrites the previous one in
# node 1's log
assert cluster.raft_exec('INCRBY', 'key', '333') == 444
cluster.wait_for_unanimity()
# Make sure log reflects the change
log.reset()
log.read()
assert match(r'.*INCRBY.*333', str(log.entries[-1].data()))
def test_append_entries():
l1 = RaftLog()
assert l1.measure_log() == (-1, 0)
assert l1.append_entries(0, None, [LogEntry(1, 1)])
assert l1 == [LogEntry(1, 1)]
# do the same thing again
assert l1.append_entries(0, None, [LogEntry(1, 1)])
assert l1 == [LogEntry(1, 1)]
# skip index not allowed
assert not l1.append_entries(2, 1, [LogEntry(1, 1)])
assert l1.measure_log() == (1, 1)
assert not l1.append_entries(1, 0, [LogEntry(0, 0)])
assert not l1.append_entries(2, 1, [LogEntry(0, 0)])
assert l1 == [LogEntry(1, 1)]
assert l1.append_entries(1, 1, [LogEntry(1, 1)])
assert l1 == [LogEntry(1, 1), LogEntry(1, 1)]
assert l1.measure_log() == (1, 2)
assert l1.append_entries(1, 1, [LogEntry(1, 2)])
assert l1 == [LogEntry(1, 1), LogEntry(1, 2)]
assert l1.append_entries(0, None, [LogEntry(1, 3)])
assert l1 == [LogEntry(1, 3)]
assert l1.append_entries(0, None, [LogEntry(1, 4), LogEntry(1, 5)])
assert l1 == [LogEntry(1, 4), LogEntry(1, 5)]
# adding entries that don't satisfy the non-decreasing terms
with pytest.raises(RaftLogInconsistent):
l1.append_entries(1, 1, [LogEntry(2, 6), LogEntry(1, 5)])
# Note: we detect this inconsistency, but don't attempt a fix
with pytest.raises(RaftLogInconsistent):
l1._invariant()
# reset l1
assert l1.append_entries(0, None, [LogEntry(1, 4), LogEntry(1, 5)])
assert l1 == [LogEntry(1, 4), LogEntry(1, 5)]
# added entries satisfy the non-decreasing terms but not in combination
# with what is already there
with pytest.raises(RaftLogInconsistent):
l1.append_entries(1, 1, [LogEntry(0, 6), LogEntry(2, 5)])
# reset l1
assert l1.append_entries(0, None, [LogEntry(1, 4), LogEntry(1, 5)])
assert l1 == [LogEntry(1, 4), LogEntry(1, 5)]
assert l1.append_entries(1, 1, [LogEntry(2, 6), LogEntry(2, 5)])
assert l1.measure_log() == (2, 3)
assert l1.append_entries(3, 2, [])
assert l1.measure_log() == (2, 3)
assert not l1.append_entries(4, 2, [])
assert not l1.append_entries(3, 1, [])
# the paper doesn't quite seem to specify if the truncate should
# happen in this case, but I believe it does no harm
assert l1.append_entries(2, 2, [])
assert l1.measure_log() == (2, 2)
def test_from_list():
entries = [LogEntry(1, 1), LogEntry(2, 3)]
l1 = RaftLog.from_list(entries)
assert l1 == entries
def test_figure7_tests():
"""suggested tests"""
entry_to_add = LogEntry(8, "x")
index_to_add = 11
prev_term_to_add = 6
add_args = (index_to_add, prev_term_to_add, [entry_to_add])
l_a = RaftLog()
l_a.append_entries(0, None,
[LogEntry(i, i) for i in [1, 1, 1, 4, 4, 5, 5, 6, 6]])
assert not l_a.append_entries(*add_args)
l_b = RaftLog()
l_b.append_entries(0, None, [LogEntry(i, i) for i in [1, 1, 1, 4]])
assert not l_b.append_entries(*add_args)
l_c = RaftLog()
l_c.append_entries(
0, None, [LogEntry(i, i) for i in [1, 1, 1, 4, 4, 5, 5, 6, 6, 6, 6]])
assert l_c.append_entries(*add_args)
l_d = RaftLog()
l_d.append_entries(
0, None, [LogEntry(i, i) for i in [1, 1, 1, 4, 4, 5, 5, 6, 6, 7, 7]])
assert not l_d.append_entries(*add_args)
l_e = RaftLog()
l_e.append_entries(0, None, [LogEntry(i, i) for i in [1, 1, 1, 4, 4, 4]])
assert not l_e.append_entries(*add_args)
l_f = RaftLog()
l_f.append_entries(
0, None, [LogEntry(i, i) for i in [1, 1, 1, 2, 2, 2, 3, 3, 3, 3, 3]])
assert not l_f.append_entries(*add_args)
def test_follower_interaction():
"""Note: this test does look at the internals of the follower. This
could possibly be avoided, but the gains of that are not clear to
me yet.
"""
config = gen_config(3)
follower_id = 1
leader_id = 0
follower = (RaftServer(follower_id, config)._set_log(RaftLog.from_list(
[]))._set_follower())
assert follower.currentTerm == 0
assert follower.commitIndex == 0
assert follower.leaderId is None
send_message(
follower,
Message(
leader_id,
follower_id,
1,
AppendEntries(leader_id, 0, -1, [LogEntry(1, 1)], 0),
),
)
step_server(follower)
assert follower.log == [LogEntry(1, 1)]
assert follower.currentTerm == 1
assert follower.commitIndex == 0
assert follower.leaderId == leader_id
assert get_message(follower).payload == AppendEntriesReply(True, 1, -1)
## checking idempotence; doing the above again
send_message(
follower,
Message(
leader_id,
follower_id,
1,
AppendEntries(leader_id, 0, -1, [LogEntry(1, 1)], 0),
),
)
step_server(follower)
assert follower.log == [LogEntry(1, 1)]
assert follower.currentTerm == 1
assert follower.commitIndex == 0
assert follower.leaderId == leader_id
assert get_message(follower).payload == AppendEntriesReply(True, 1, -1)
# message indicating that the log entry is now committed
send_message(
follower,
Message(leader_id, follower_id, 1,
AppendEntries(leader_id, 1, 1, [], 1)),
)
step_server(follower)
assert follower.log == [LogEntry(1, 1)]
assert follower.currentTerm == 1
assert follower.commitIndex == 1 # this is the update
assert follower.leaderId == leader_id
assert get_message(follower).payload == AppendEntriesReply(True, 1, -1)
# now lengthen the log; one new entry is already commited the other not yet
send_message(
follower,
Message(
leader_id,
follower_id,
1,
AppendEntries(leader_id, 1, 1,
[LogEntry(1, 2), LogEntry(1, 3)], 2),
),
)
step_server(follower)
assert follower.log == [LogEntry(1, 1), LogEntry(1, 2), LogEntry(1, 3)]
assert follower.currentTerm == 1
assert follower.commitIndex == 2
assert follower.leaderId == leader_id # and hence has been updated
assert get_message(follower).payload == AppendEntriesReply(True, 3, -1)
# new leader, makes some changes to the log
leader_id = 2
send_message(
follower,
Message(
leader_id,
follower_id,
2,
AppendEntries(leader_id, 2, 1, [LogEntry(2, 3)], 3),
),
)
step_server(follower)
assert follower.log == [LogEntry(1, 1), LogEntry(1, 2), LogEntry(2, 3)]
assert follower.currentTerm == 2
assert follower.commitIndex == 3
assert get_message(follower).payload == AppendEntriesReply(True, 3, -1)
# send append entries from leader with old term
leader_id = 2
send_message(
follower,
Message(
leader_id,
follower_id,
0,
AppendEntries(leader_id, 2, 1, [LogEntry(2, 3)], 3),
),
)
step_server(follower)
assert follower.log == [LogEntry(1, 1), LogEntry(1, 2), LogEntry(2, 3)]
assert follower.currentTerm == 2
assert follower.commitIndex == 3
assert get_message(follower).payload == AppendEntriesReply(False, -1, 2)
def test_leader_interaction():
"""Note: this test does look at the internals of the leader. This
could possibly be avoided, but the gains of that are not clear to
me yet.
"""
config = gen_config(3)
follower_ids = [1, 2]
leader_id = 0
leader = (RaftServer(leader_id,
config)._set_log(RaftLog.from_list([LogEntry(1, 1)
]))._set_leader())
assert all(leader.state.nextIndex[i] == 1 for i in follower_ids)
assert all(leader.state.matchIndex[i] == 0 for i in follower_ids)
step_server(leader)
ms = get_all_messages(leader)
assert [m.recver_id for m in ms] == follower_ids
assert all(m.payload == AppendEntries(leader_id, 1, 1, [], 0) for m in ms)
assert all(leader.state.matchIndex[i] == 0 for i in follower_ids)
assert all(leader.state.nextIndex[i] == 1 for i in follower_ids)
# send to everyone again
step_server(leader)
ms = get_all_messages(leader)
assert [m.recver_id for m in ms] == follower_ids
send_message(leader, Message(1, 0, 0, AppendEntriesReply(False, -1, 1)))
step_server(leader)
ms = get_all_messages(leader)
assert [m.recver_id for m in ms] == follower_ids
assert all(leader.state.matchIndex[i] == 0 for i in follower_ids)
assert leader.state.nextIndex[1] == 0
assert leader.state.nextIndex[2] == 1
msd = {m.recver_id: m.payload for m in ms}
assert msd[1] == AppendEntries(leader_id, 0, -1, [LogEntry(1, 1)], 0)
assert msd[2] == AppendEntries(leader_id, 1, 1, [], 0)
send_message(leader, Message(1, 0, 0, AppendEntriesReply(True, 1, -1)))
step_server(leader)
ms = get_all_messages(leader)
assert [m.recver_id for m in ms] == [
1,
2,
] # 1 now synched, but send empty appendEntries
assert leader.state.matchIndex[1] == 1
assert leader.state.matchIndex[2] == 0
assert leader.state.nextIndex[1] == 1
assert leader.state.nextIndex[2] == 1
# assume leader had some entries added
leader.log.append_entries(1, 1, [LogEntry(1, 2), LogEntry(1, 3)])
assert leader.log == [LogEntry(1, 1), LogEntry(1, 2), LogEntry(1, 3)]
step_server(leader)
ms = get_all_messages(leader)
assert [m.recver_id for m in ms] == follower_ids
assert leader.state.matchIndex[1] == 1
assert leader.state.matchIndex[2] == 0
msd = {m.recver_id: m.payload for m in ms}
assert msd[1] == AppendEntries(
leader_id, 1, 1, [LogEntry(1, 2), LogEntry(1, 3)], 0)
# leader still has not heard from 2, so nextIndex not yet lowered
assert msd[2] == AppendEntries(
leader_id, 1, 1, [LogEntry(1, 2), LogEntry(1, 3)], 0)
# tell leader we can't add the entry at index 1
send_message(leader, Message(2, 0, 0, AppendEntriesReply(False, -1, 1)))
step_server(leader)
ms = get_all_messages(leader)
assert [m.recver_id for m in ms] == follower_ids
assert leader.state.matchIndex[1] == 1
assert leader.state.matchIndex[2] == 0
msd = {m.recver_id: m.payload for m in ms}
print(msd)
assert msd[1] == AppendEntries(
leader_id, 1, 1, [LogEntry(1, 2), LogEntry(1, 3)], 0)
# leader still has not heard from 2, so nextIndex not yet lowered
assert msd[2] == AppendEntries(
leader_id, 0, -1,
[LogEntry(1, 1), LogEntry(1, 2),
LogEntry(1, 3)], 0)
print(msd)
assert msd[1] == AppendEntries(
leader_id, 1, 1, [LogEntry(1, 2), LogEntry(1, 3)], 0)
# leader still has not heard from 2, so nextIndex not yet lowered
assert msd[2] == AppendEntries(
leader_id, 0, -1,
[LogEntry(1, 1), LogEntry(1, 2),
LogEntry(1, 3)], 0)
@pytest.mark.parametrize(
"log_leader, log_follower",
[
(
RaftLog.from_list([LogEntry(1, 1),
LogEntry(2, 2),
LogEntry(3, 3)]),
RaftLog.from_list([]),
),
(
RaftLog.from_list([LogEntry(1, 1),
LogEntry(2, 2),
LogEntry(3, 3)]),
RaftLog.from_list([LogEntry(1, 1),
LogEntry(1, 2),
LogEntry(1, 2)]),
),
(
RaftLog.from_list([LogEntry(1, 1), LogEntry(2, 2)]),
RaftLog.from_list([
LogEntry(1, 1),