def recover(self):
"""
Recover a Raft Node.
"""
self.alive = True
debug_test("Raft Node", self.addr, "Recovered")
debug_log_raft("Raft Node", self.addr, "Recovered")
async def get_value(self, key: str):
"""
Get a value from the RAFT cluster
"""
debug_log_raft("Raft", self.addr, "Getting key", key)
assert (self.role == Role.LEADER.value)
value = await self.persistent_storage.get(key)
return value
def clear_election_timer(self):
"""
Clear the election timer.
"""
assert (self.election_timer != None)
self.election_timer.cancel()
debug_log_raft(self.addr, "cancelling election timer")
self.election_timer = None
def become_follower(self):
"""
Become a follower.
"""
debug_log_raft("RaftNode ", self.addr, ": BECOME FOLLOWER")
self.set_role(Role.FOLLOWER)
self.next_index = None
self.match_index = None
self.voted_for = None
def become_leader(self):
debug_log_raft("Node ", self.addr, ": NEW LEADER")
self.set_role(Role.LEADER)
# When a leader first comes to power,
# it initializes all nextIndex values to the index just after the
# last one in its log
self.match_index = {p: -1 for p in self.peers}
self.next_index = {p: self.log.latest_index() + 1 for p in self.peers}
self.clear_election_timer()
self.start_new_append_entries()
def shutdown(self):
"""
Shutdown a Raft Node.
"""
if self.alive:
self.alive = False
debug_test("Raft Node", self.addr, "No longer alive")
debug_log_raft("Raft Node", self.addr, "no longer alive")
self.cancel_idle_heartbeat()
for cancel_handler in self.handlers:
cancel_handler()
sys.exit()
async def get_vote_from_peer(self, peer):
"""
Get a vote from a peer for the current election cycle.
"""
debug_log_raft("RAFT ", self.addr, "REQUESTING VOTES FROM ", peer)
condition = lambda msg: (msg.has_type(MessageType.VOTE) and msg.src ==
peer)
vote_response = self.async_scheduler.get_message_wait_future(condition)
self.send_REQUEST_VOTE(peer)
vote_msg = await vote_response
term, vote_granted = vote_msg.get_args(['term', 'vote_granted'])
if term > self.current_term:
self.step_down()
return
return vote_granted, peer
async def reconcile_logs_with_peers(self):
"""
Reconcile our log with all our peers.
"""
debug_log_raft("Raft", self.addr, ": Reconciling logs with peers")
# debug_test("Raft", self.addr, ": sending out APPEND_ENTRIES to peers")
try:
peer_tasks = [
asyncio.create_task(
self.reconcile_logs_with_append_entries(peer))
for peer in self.peers
]
# debug_log_raft("Raft", self.addr, ": reconciling logs with peers", peer_tasks)
await asyncio.wait(peer_tasks)
finally:
for task in peer_tasks:
task.cancel()
async def set_match_index(self, peer, match_index):
"""
Set the match index for a given node.
"""
debug_log_raft("RAFT NODE", self.addr, "role value", self.role,
"setting match index", peer, match_index)
assert (self.role == Role.LEADER.value)
self.match_index[peer] = match_index
while True:
# • If there exists an N such that N > commitIndex, a majority
# of matchIndex[i] ≥ N, and log[N].term == currentTerm:
# set commitIndex = N (§5.3, §5.4).
if (1 + sum(self.match_index[match] >= (self.commit_index + 1)
for match in self.match_index)) / (len(self.peers) +
1) >= .5:
debug_log_raft("Raft", self.addr, "Setting commit index to be",
self.commit_index + 1)
await self.set_commit_index(self.commit_index + 1)
else:
return
def set_election_timer(self):
"""
Start an election timeout timer.
If we were a candidate for an election, we time out the election and start a new election timeout.
"""
debug_log_raft("Raft", self.addr, "restarting election timeout")
if (self.election_timer != None):
# print("cancelinng election timer")
self.election_timer.cancel()
async def timeout():
time_to_sleep = uniform(config.ELECTION_TIMEOUT_MIN,
config.ELECTION_TIMEOUT_MAX)
await asyncio.sleep(time_to_sleep)
if self.candidacy != None:
self.candidacy.cancel()
# Can't figure out why this isn't stopping so let's just not try and elect if we are the main guy.
if not self.role == Role.LEADER.value:
self.candidacy = asyncio.create_task(self.start_canidacy())
self.election_timer = asyncio.create_task(timeout())
async def request_votes_and_get_majority(self):
"""
Request votes from all your peers for the current election cycle and don't stop until we get a majority!
"""
debug_log_raft("RaftNode", self.addr, ": requesting votes from peers:",
self.peers)
# We assume that self.vote_request_task is None for every peer.
vote_request_tasks = [
self.get_vote_from_peer(peer) for peer in self.peers
]
have_majority = False
num_votes = 1
while not have_majority:
finished, unfinished = await asyncio.wait(
vote_request_tasks, return_when=asyncio.FIRST_COMPLETED)
for finished_task in finished:
vote, peer = finished_task.result()
debug_log_raft("RaftNode", self.addr, ":Got a vote of", vote,
"from peer", peer)
if vote == True:
num_votes += 1
debug_log_raft("RaftNode", self.addr, ":We now have ", num_votes,
" votes majority is ",
num_votes / (len(self.peers) + 1))
# If we have a majority.
if num_votes / (len(self.peers) + 1) >= .5:
have_majority = True
async def set_commit_index(self, commit_index):
"""
Go through and apply everything between self.last_applied and new commit index
"""
while self.last_applied < commit_index:
self.last_applied += 1
debug_log_raft("RAFT NODE", self.addr, "APPLYING LOG ENTRY",
self.last_applied, self.log.entries)
entry = self.log.entries[self.last_applied]
# Apply the args
args = entry.args
for key in args:
debug_log_raft("RAFT NODE", self.addr, "APPLYING LOG ENTRY",
self.last_applied, "setting", key, "to",
args[key])
debug_test("Raft", self.addr, ": Applying log entry",
self.last_applied, "setting", key, "to", args[key])
await self.persistent_storage.set(key, args[key])
self.commit_index = commit_index
# Resolve any waiters if their request has been committed.
for commit_num, fut in self.commit_index_waiters:
if commit_num >= self.commit_index:
debug_log_raft("Resolving set request with num ", commit_num)
fut.set_result(True)
async def reconcile_logs_with_append_entries(self, peer):
"""
Send an append_entries RPC to `peer` to reconcile our logs.
"""
if not self.alive:
return
debug_log_raft("Raft", self.addr, ": reconciling log with", peer)
accept = False
while not accept:
if not self.alive:
return
append_entries_msg, log_size_to_send = self.make_append_entries_msg(
peer)
response = await self.send_message_and_await(
append_entries_msg, lambda msg:
(msg.has_type(MessageType.APPEND_ENTRIES_REPLY) and msg.src ==
peer and ('prev_log_index' in msg.args) and
(msg.args['prev_log_index'] == append_entries_msg.args[
'prev_log_index']) and ('entries_len' in msg.args) and
(msg.args['entries_len'] == len(append_entries_msg.args[
'entries']))))
debug_log_raft("Raft", self.addr,
": got response reconciling log with", peer,
response)
term, accept = response.get_args(['term', 'accept'])
debug_log_raft("Raft", self.addr,
": got response reconciling log with", peer, "term",
term, "accept", accept)
if accept:
debug_test("Raft", self.addr, ": Reconciling log with", peer,
"complete")
# We're behind
if self.current_term < term:
self.step_down(term)
return
if not accept:
self.next_index[peer] -= 1
else:
# debug_log_raft("Raft", self.addr, ": got response reconciling log", peer," with matching up to ", log_size_to_send)
self.next_index[peer] = log_size_to_send
await self.set_match_index(peer, log_size_to_send - 1)
async def start_canidacy(self):
"""
Run a campaign to be the leader.
"""
debug_log_raft("RaftNode ", self.addr, ": Starts Canidacy")
#1.Follower increments Term
self.current_term += 1
#1. Transition to Canidate State
# Clear all tasks requesting a vote.
self.become_candidate()
debug_log_raft("RaftNode ", self.addr,
": Successful Start Canidacy 00")
#2. Vote for Ourself
self.voted_for = self.addr
# self.votes += 1/app
#3. Sends Vote Reqeust to All Peers
debug_log_raft("RaftNode ", self.addr, ": REQUESTING VOTES")
await self.request_votes_and_get_majority()
debug_log_raft(
"Becoming the leader because we got a majority of votes.")
self.become_leader()
async def handle_append_entries(self, append_entries_msg: Message):
"""
Handle an append_entries RPC from a leader to reconcile our log.
"""
if not self.alive:
return
# debug_log_raft("Raft", self.addr, ": handling append_entries")
(term, prev_log_index, prev_log_term, entries,
leader_commit) = append_entries_msg.get_args([
'term', 'prev_log_index', 'prev_log_term', 'entries',
'leader_commit'
])
entries = [LogEntry(entry['term'], entry['args']) for entry in entries]
if self.current_term < term:
# We're behind
self.step_down(term)
else:
# We got a message from the leader, so we reset the election timer.
self.set_election_timer()
assert (self.role != Role.LEADER.value)
# We are now a follower (in case we were a candidate before)
if self.role == Role.CANDIDATE.value:
self.become_follower()
prev_log_matches = prev_log_term is None or \
self.log.term_at_index(prev_log_index) == prev_log_term
if (term < self.current_term or (not prev_log_matches)):
self.respond_to_append_entries(append_entries_msg, False)
return
"""
If an existing entry conflicts with a new one (same index
but different terms), delete the existing entry and all that
follow it (§5.3). Append any new entries not in the log.
"""
index_in_our_log = prev_log_index + 1
for entry in entries:
entry_term = entry.term
log_term = self.log.term_at_index(index_in_our_log)
if (log_term == None):
# If there's no entry at this index, we append the entry
self.log.append_entry(entry)
elif (log_term != entry_term):
# If the entry at this point is different, we are behind and so
# we remove every entry after this and copy the entries given by the leader
self.log.remove_entries_at_and_after(index_in_our_log)
self.log.append_entry(entry)
index_in_our_log += 1
# self.log.entries = self.log.entires[:prev_log_index + entries_after_prev_log_entry] + entries[entries_after_prev_log_entry - 1:]
# self.log.replace_logs_after_index(prev_log_index + entries_after_prev_log_entry, entries[entries_after_prev_log_entry - 1:])
# last_matching_index = prev_log_index + entries_after_prev_log_entry
if leader_commit > self.commit_index:
new_commit = min(leader_commit, self.log.latest_index())
await self.set_commit_index(new_commit)
self.respond_to_append_entries(append_entries_msg, True)
debug_log_raft("RAFT", self.addr, "new log entries", self.log.entries,
"from", append_entries_msg)
def become_candidate(self):
debug_log_raft("RaftNode ", self.addr, ": Becoming Candidate")
self.set_role(Role.CANDIDATE)
self.set_election_timer()