def __init__(self, server_no, num_servers, state_machine, persist=False):
self.server_no = server_no
self._logger = logging.getLogger(f"RaftServer-{server_no}")
self.num_servers = num_servers
self._state_machine = state_machine
self.persist = persist
self.server_state_path = f'server-{server_no}.state'
# Persistent state
self._term = 0
self.voted_for = None
self.log = Log()
self._commit_index = -1
self.last_applied = -1
self.leader_id = None
self.received_votes = set()
self.state = State.FOLLOWER
self.outbox = queue.Queue()
# volatile leader state
self.next_index = None
self.match_index = None
self.unhandled_client_requests = {}
self.recover()
def __init__(self, conf):
self.role = "follower"
self.id = conf["id"]
self.addr = conf["addr"]
self.peers = conf["peers"]
# 持久化保存的状态
self.current_term = 0
self.voted_for = None
# 建立节点日志文件夹
if not os.path.exists(self.id):
os.mkdir(self.id)
# 初始化并读取日志
self.load()
self.log = Log(self.id)
# 非持久保存的状态
# 已经提交的日志索引
self.commit_index = 0
# 已经执行的日志索引
self.last_applied = 0
# Leader初始化
# 下一条要复制的日志
self.next_index = {_id: self.log.last_log_index + 1 for _id in self.peers}
# follower与leader最后一条匹配的日志
self.match_index = {_id: -1 for _id in self.peers}
# 添加日志条目
self.leader_id = None
# 请求投票
self.vote_ids = {_id: 0 for _id in self.peers}
self.client_addr = None
# 时钟
self.wait_ms = (10, 20)
self.next_leader_election_time = time.time() + random.randint(*self.wait_ms)
self.next_heartbeat_time = 0
# 消息
self.ss = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.ss.bind(self.addr)
self.ss.settimeout(2)
self.cs = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def test_increment_commit_index_calls_apply(no_network_raft_follower,
log_entry):
no_network_raft_follower.log = Log.from_entries([log_entry])
with mock.patch.object(no_network_raft_follower._state_machine,
'apply') as mocked_apply:
no_network_raft_follower.commit_index = 0
assert mocked_apply.called
def test_log_append_entries_conflict():
log = Log()
entry = LogEntry(term=1, msg=42)
log.append(log_index=0, prev_log_term=0, entry=LogEntry(term=0, msg=42))
log.append(log_index=1, prev_log_term=0, entry=LogEntry(term=0, msg=42))
log.append(log_index=2, prev_log_term=0, entry=LogEntry(term=0, msg=42))
log.append(log_index=0, prev_log_term=0, entry=entry)
assert log[0] == entry
assert len(log) == 1
def filled_log():
log = Log()
log.append(log_index=0, prev_log_term=0, entry=LogEntry(term=0, msg=0))
log.append(log_index=1, prev_log_term=0, entry=LogEntry(term=0, msg=1))
log.append(log_index=2, prev_log_term=0, entry=LogEntry(term=0, msg=2))
log.append(log_index=3, prev_log_term=0, entry=LogEntry(term=1, msg=3))
log.append(log_index=4, prev_log_term=1, entry=LogEntry(term=1, msg=4))
log.append(log_index=5, prev_log_term=1, entry=LogEntry(term=1, msg=5))
assert len(log) == 6
return log
def test_log_wrong_term():
log = Log()
log.append(log_index=0, prev_log_term=0, entry=LogEntry(term=0, msg=42))
log.append(log_index=1, prev_log_term=0, entry=LogEntry(term=0, msg=42))
with pytest.raises(LogDifferentTermError):
entry = LogEntry(term=1, msg=42)
log.append(log_index=2, prev_log_term=1, entry=entry)
def test_figure_7(raft_cluster):
leader, a, b, c, d, e, f = raft_cluster(7)
leader_log_entries = [
LogEntry(1, "1"),
LogEntry(1, "2"),
LogEntry(1, "3"),
LogEntry(4, "4"),
LogEntry(4, "5"),
LogEntry(5, "6"),
LogEntry(5, "7"),
LogEntry(6, "8"),
LogEntry(6, "9"),
LogEntry(6, "10"),
]
leader.log = Log.from_entries(leader_log_entries)
a.log = Log.from_entries(leader_log_entries[:-1])
b.log = Log.from_entries(leader_log_entries[:4])
c.log = Log.from_entries(leader_log_entries + [LogEntry(6, "11")])
d.log = Log.from_entries(
leader_log_entries + [LogEntry(7, "11"), LogEntry(7, "11")]
)
e.log = Log.from_entries(
leader_log_entries[:5] + [LogEntry(4, "6"), LogEntry(4, "7")]
)
f_log_entries = [
LogEntry(2, "4"),
LogEntry(2, "5"),
LogEntry(2, "6"),
LogEntry(3, "7"),
LogEntry(3, "8"),
LogEntry(3, "9"),
LogEntry(3, "10"),
LogEntry(3, "11"),
]
f.log = Log.from_entries(leader_log_entries[:3] + f_log_entries)
with mock.patch.object(leader, 'leader_log_append') as mocked:
leader.become_leader()
replicate(leader, a)
replicate(leader, b)
replicate(leader, c)
replicate(leader, d)
replicate(leader, e)
replicate(leader, f)
assert logs_same(leader.log, a.log)
assert logs_same(leader.log, b.log)
assert not logs_same(leader.log, c.log)[0]
assert len(c.log) - len(leader.log) == 1
assert not logs_same(leader.log, d.log)[0]
assert len(d.log) - len(leader.log) == 2
assert logs_same(leader.log, e.log)
assert logs_same(leader.log, f.log)
def test_log_append_no_truncate():
"""
If an existing entry conflicts with a new one (same index but different terms), delete the existing entry and all
that follow it.
The if here is crucial. If the follower has all the entries the leader sent, the follower MUST NOT truncate its log.
Any elements following the entries sent by the leader MUST be kept. This is because we could be receiving an
outdated AppendEntries RPC from the leader, and truncating the log would mean “taking back” entries that we may
have already told the leader that we have in our log.
"""
log = Log()
entry = LogEntry(term=0, msg=42)
log.append(log_index=0, prev_log_term=0, entry=entry)
log.append(log_index=1, prev_log_term=0, entry=entry)
log.append(log_index=2, prev_log_term=0, entry=entry)
log.append(log_index=0, prev_log_term=0, entry=entry)
assert log[0] == entry
assert len(log) == 3
def test_log_wrong_index():
log = Log()
with pytest.raises(LogNotCaughtUpError):
entry = LogEntry(term=0, msg=42)
log.append(log_index=1, prev_log_term=0, entry=entry)
def test_log_append_invalid_entry():
log = Log()
with pytest.raises(ValueError):
log.append(log_index=0, prev_log_term=0, entry=1)
assert len(log) == 0
def test_log_append_none():
log = Log()
log.append(log_index=0, prev_log_term=0, entry=None)
assert len(log) == 0
def test_log_append_correct():
log = Log()
entry = LogEntry(term=0, msg=42)
log.append(log_index=0, prev_log_term=0, entry=entry)
assert log[0] == entry
class Node(object):
def __init__(self, conf):
self.role = "follower"
self.id = conf["id"]
self.addr = conf["addr"]
self.peers = conf["peers"]
# 持久化保存的状态
self.current_term = 0
self.voted_for = None
# 建立节点日志文件夹
if not os.path.exists(self.id):
os.mkdir(self.id)
# 初始化并读取日志
self.load()
self.log = Log(self.id)
# 非持久保存的状态
# 已经提交的日志索引
self.commit_index = 0
# 已经执行的日志索引
self.last_applied = 0
# Leader初始化
# 下一条要复制的日志
self.next_index = {_id: self.log.last_log_index + 1 for _id in self.peers}
# follower与leader最后一条匹配的日志
self.match_index = {_id: -1 for _id in self.peers}
# 添加日志条目
self.leader_id = None
# 请求投票
self.vote_ids = {_id: 0 for _id in self.peers}
self.client_addr = None
# 时钟
self.wait_ms = (10, 20)
self.next_leader_election_time = time.time() + random.randint(*self.wait_ms)
self.next_heartbeat_time = 0
# 消息
self.ss = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.ss.bind(self.addr)
self.ss.settimeout(2)
self.cs = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def load(self):
file_path = self.id + "/key.json"
if os.path.exists(file_path):
with open(file_path, "r") as f:
data = json.load(f)
self.current_term = data["current_term"]
self.voted_for = data["voted_for"]
else:
self.save()
def save(self):
data = {
"current_term": self.current_term,
"voted_for": self.voted_for,
}
file_path = self.id + "/key.json"
logging.info("debug: " + str(data["current_term"]) + " " + str(data["voted_for"]))
with open(file_path, "w") as f:
json.dump(data, f)
def send(self, msg, addr):
msg = json.dumps(msg).encode("utf-8")
self.cs.sendto(msg, addr)
def recv(self):
msg, addr = self.ss.recvfrom(65535)
return json.loads(msg), addr
def redirect(self, data, addr):
if data is None:
return None
if data["type"] == "client_append_entries":
# 若当前节点不是leader, 将请求重定向到leader节点
if self.role != "leader":
if self.leader_id:
logging.info("redirect: client_append_entries to leader")
return None
else:
self.client_addr = addr
return data
if data["dst_id"] != self.id:
logging.info("redirect: to " + data["dst_id"])
self.send(data, self.peers[data["dst_id"]])
return None
else:
return data
# append entries rpc
def append_entries(self, data):
"""
对leader复制日志条目的响应
:param data:
:return:
"""
response = {
"type": "append_entries_response",
"src_id": self.id,
"dst_id": data["src_id"],
"term": self.current_term,
"success": False,
}
# 如果接收到的消息term比当前节点更小, 拒绝leader
if data["term"] < self.current_term:
logging.info(" 2. smaller term")
logging.info(" 3. success = False: smaller term")
logging.info(" 4. send append_entries_response to leader " + data["src_id"])
response["success"] = False
self.send(response, self.peers[data["src_id"]])
return
self.leader_id = data["leader_id"]
# 不含新日志条目的heartbeat
if data["entries"] is None:
logging.info(" 4. heartbeat")
return
prev_log_index = data["prev_log_index"]
prev_log_term = data["prev_log_term"]
local_prev_log_term = self.log.get_log_term(prev_log_index)
# 如果要求复制的日志与该节点保存的最新一条日志不匹配, 复制失败
if local_prev_log_term != prev_log_term:
logging.info(" 4. success = False: index not match or term not match")
logging.info(" 5. send append_entries_response to leader " + data["src_id"])
logging.info(" 6. log delete_entries")
logging.info(" 7. log save")
response["success"] = False
self.send(response, self.peers[data["src_id"]])
# 将与leader节点不一致的日志删除
self.log.delete_entries(prev_log_index)
else:
logging.info(" 4. success = True")
logging.info(" 5. send append_entries_response to leader " + data["src_id"])
logging.info(" 6. log append_entries")
logging.info(" 7. log save")
# 发送追加成功的消息
# 并添加日志条目
response["success"] = True
self.send(response, self.peers[data["src_id"]])
self.log.append_entries(prev_log_index, data["entries"])
# 如果已提交日志比当前节点已提交日志的索引更大
# 当前节点更新提交日志索引
leader_commit = data["leader_commit"]
if leader_commit > self.commit_index:
commit_index = min(leader_commit, self.log.last_log_index)
logging.info(" 8. commit_index " + str(commit_index))
return
def request_vote(self, data):
"""
响应request_vote消息
:param data:
:return:
"""
response = {
"type": "request_vote_response",
"src_id": self.id,
"dst_id": data["src_id"],
"term": self.current_term,
"vote_granted": False
}
# 如果请求投票的candidate term比当前节点更小, 拒绝投票
if data["term"] < self.current_term:
logging.info(" 2. smaller term")
logging.info(" 3. success = False")
logging.info(" 4. send request_vote_response to candidate " + data["src_id"])
response["vote_granted"] = False
self.send(response, self.peers[data['src_id']])
return
logging.info(" 2. same term")
candidate_id = data["candidate_id"]
last_log_index = data["last_log_index"]
last_log_term = data["last_log_term"]
# 当节点未投票或已经投票给发来请求的节点
if self.voted_for is None or self.voted_for == candidate_id:
# 判断发送请求的节点日志是否比本节点的日志更新
if last_log_index >= self.log.last_log_index and last_log_term >= self.log.last_log_term:
self.voted_for = data["src_id"]
# 持久化保存
self.save()
# 同意投票
response["vote_granted"] = True
self.send(response, self.peers[data["src_id"]])
logging.info(" 3. success = True: candidate log is newer")
logging.info(" 4. send request_vote_response to candidate " + data["src_id"])
# 否则, 拒绝节点的投票请求
else:
self.voted_for = None
self.save()
response["vote_granted"] = False
self.send(response, self.peers[data["src_id"]])
logging.info(" 3. success = False: candidate log is older")
logging.info(" 4. send request_vote_response to candidate " + str(data["src_id"]))
# 节点已经投票过
else:
response["vote_granted"] = False
self.save()
self.send(response, self.peers[data["src_id"]])
logging.info(" 3. success = False: has voted for " + self.voted_for)
logging.info(" 4. send request_vote_response to candidate " + str(data["src_id"]))
return
def all_do(self, data):
logging.info("----------------all----------------")
# 应用已提交的日志
if self.commit_index > self.last_applied:
self.last_applied = self.commit_index
logging.info("all: 1. last_applied = " + str(self.last_applied))
if data is None:
return
if data["type"] == "client_append_entries":
# follower节点不响应client的请求
# TODO 重定向到leader
return
# 若接收到一个从更新的节点发来的消息, 退化为follower并重置选举状态
if data["term"] > self.current_term:
logging.info("all: 1. bigger term")
logging.info(" 2. become follower")
self.role = "follower"
self.current_term = data["term"]
self.voted_for = data["src_id"]
self.save()
return
def follower_do(self, data):
"""
follower节点应完成的功能
:param data:
:return:
"""
logging.info("----------------follower----------------")
t = time.time()
if data is not None:
# 处理append_entries消息
if data["type"] == "append_entries":
logging.info("follower: 1. recv append_entries from leader " + data["src_id"])
# 若term合法, 则接受这条日志
if data["term"] == self.current_term:
logging.info(" 2. same term")
logging.info(" 3. reset next_election_time")
# 重置选举超时时间
self.next_leader_election_time = t + random.randint(*self.wait_ms)
# 添加日志
self.append_entries(data)
# 处理选举消息
elif data["type"] == "request_vote":
logging.info("follower: 1. recv request_vote from candidate " + data["src_id"])
self.request_vote(data)
# 经过选举超时间隔, 未收到leader消息, 转为candidate
if t > self.next_leader_election_time:
logging.info("follower: 1. become candidate")
self.next_leader_election_time = t + random.randint(*self.wait_ms)
self.role = "candidate"
# 给自己投票
self.current_term += 1
self.voted_for = self.id
self.save()
self.vote_ids = {_id: 0 for _id in self.peers}
return
def candidate_do(self, data):
"""
candidate节点完成的功能
:param data:
:return:
"""
logging.info("----------------candidate----------------")
t = time.time()
# 向所有节点广播request_vote
for dst_id in self.peers:
if self.vote_ids[dst_id] == 0:
logging.info("candidate: 1. send request_vote to peer " + dst_id)
request = {
"type": "request_vote",
"src_id": self.id,
"dst_id": dst_id,
"term": self.current_term,
"candidate_id": self.id,
"last_log_index": self.log.last_log_index,
"last_log_term": self.log.last_log_term,
}
# 发送请求选举
self.send(request, self.peers[dst_id])
if data is not None and data["term"]:
# 处理response
if data["type"] == "request_vote_response":
logging.info("candidate: 1. recv request_vote_response from follower " + data["src_id"])
self.vote_ids[data["src_id"]] = data["vote_granted"]
vote_count = sum(list(self.vote_ids.values()))
# 节点获得多数派支持即成为新的leader
if vote_count >= len(self.peers) // 2:
logging.info(" 2. become leader")
self.role = "leader"
self.voted_for = None
self.save()
self.next_heartbeat_time = 0
# 为每一个follower初始化下一条推送的日志索引和匹配的日志索引
self.next_index = {_id: self.log.last_log_index + 1 for _id in self.peers}
self.match_index = {_id: 0 for _id in self.peers}
return
# 如果收到其他节点声称自己当选为leader
elif data["type"] == "append_entries":
logging.info("candidate: 1. recv append_entries from leader " + str("src_id"))
logging.info(" 2. become follower")
self.next_leader_election_time = t + random.randint(*self.wait_ms)
self.role = "follower"
self.voted_for = None
self.save()
return
# 本轮选举未选出leader, 重新开始新一轮选举
if t > self.next_leader_election_time:
logging.info("candidate: 1. leader_election timeout")
logging.info(" 2. become candidate")
self.next_leader_election_time = t + random.randint(*self.wait_ms)
self.role = "candidate"
self.current_term += 1
self.voted_for = self.id
self.save()
self.vote_ids = {_id: 0 for _id in self.peers}
return
def leader_do(self, data):
"""
leader 必须实现的功能
:param data:
:return:
"""
logging.info("----------------leader----------------")
t = time.time()
# heartbeat信号时间必须远小于election timeout
if t > self.next_heartbeat_time:
self.next_heartbeat_time = t + random.randint(0, 5)
# 广播append entries消息
for dst_id in self.peers:
logging.info("leader: 1. send append_entries to peer " + dst_id)
request = {
"type": "append_entries",
"src_id": self.id,
"dst_id": dst_id,
"term": self.current_term,
"leader_id": self.id,
"prev_log_index": self.next_index[dst_id] - 1,
"prev_log_term": self.log.get_log_term(self.next_index[dst_id] - 1),
"entries": self.log.get_entries(self.next_index[dst_id]),
"leader_commit": self.commit_index,
}
self.send(request, self.peers[dst_id])
# 处理client请求
if data is not None and data["type"] == "client_append_entries":
data["term"] = self.current_term
# 追加日志
self.log.append_entries(self.log.last_log_index, [data])
logging.info("leader: 1. recv append_entries from client")
logging.info(" 2. log append_entries")
logging.info(" 3. log save")
return
# 处理append_entries response
if data is not None and data["term"] == self.current_term:
if data["type"] == "append_entries_response":
logging.info("leader: 1. recv append_entries_response from follower " + data["src_id"])
# 追加日志不成功
if not data["success"]:
# next_index - 1, 重试
self.next_index[data["src_id"]] -= 1
logging.info("leader: 1. recv append_entries_response from follower " + data["src_id"])
else:
# 日志复制成功, 匹配日志索引增加
self.match_index[data["src_id"]] = self.next_index[data["src_id"]]
self.next_index[data["src_id"]] = self.log.last_log_index + 1
self.match_index[data['src_id']] = self.next_index[data['src_id']]
self.next_index[data['src_id']] = self.log.last_log_index + 1
logging.info(" 2. success = True")
logging.info(
" 3. match_index = " + str(self.match_index[data['src_id']]) + " next_index = " + str(
self.next_index[data['src_id']]))
while True:
N = self.commit_index + 1
count = 0
for _id in self.peers:
if self.match_index[_id] >= N:
count += 1
# 若日志已经被复制到多数机器上, 则提交这条日志
if count >= len(self.peers) // 2:
self.commit_index = N
logging.info("leader: 1. commit + 1")
# 通知client这条日志已经被提交
if self.client_addr:
response = {
"index": self.commit_index,
}
self.send(response, (self.client_addr[0], 10000))
break
else:
logging.info("leader: 2. commit = " + str(self.match_index))
break
def run(self):
while True:
try:
try:
data, addr = self.recv()
except Exception as e:
data, addr = None, None
data = self.redirect(data, addr)
self.all_do(data)
if self.role == "follower":
self.follower_do(data)
if self.role == "candidate":
self.candidate_do(data)
if self.role == "leader":
self.leader_do(data)
except Exception as e:
logging.info(e)
break
self.ss.close()
self.cs.close()
class RaftServer:
def __init__(self, server_no, num_servers, state_machine, persist=False):
self.server_no = server_no
self._logger = logging.getLogger(f"RaftServer-{server_no}")
self.num_servers = num_servers
self._state_machine = state_machine
self.persist = persist
self.server_state_path = f'server-{server_no}.state'
# Persistent state
self._term = 0
self.voted_for = None
self.log = Log()
self._commit_index = -1
self.last_applied = -1
self.leader_id = None
self.received_votes = set()
self.state = State.FOLLOWER
self.outbox = queue.Queue()
# volatile leader state
self.next_index = None
self.match_index = None
self.unhandled_client_requests = {}
self.recover()
@property
def commit_index(self):
return self._commit_index
@commit_index.setter
def commit_index(self, value):
self._commit_index = value
while value > self.last_applied:
self.last_applied += 1
operation = self.log[self.last_applied].msg
self._state_machine.apply(operation)
if operation in self.unhandled_client_requests:
self._send(self.unhandled_client_requests[operation],
Result('ok'))
@property
def term(self):
return self._term
@term.setter
def term(self, value):
self.voted_for = None
self._term = value
def become_leader(self):
self._logger.info(
f"Transitioned to leader of term {self.term}, setting next_index to {max(len(self.log), 0)}"
)
self.state = State.LEADER
self.next_index = [
max(0, len(self.log)) for _ in range(self.num_servers)
]
self.match_index = [-1 for _ in range(self.num_servers)]
self.match_index[self.server_no] = len(self.log)
self.leader_log_append(NoOp(request_id=0))
def become_follower(self):
self.state = State.FOLLOWER
self.unhandled_client_requests = {}
def become_candidate(self):
self.state = State.CANDIDATE
self.term += 1
self._logger.info(f"Transitioned to candidate in term {self.term}")
self.voted_for = self.server_no
self.received_votes.add(self.server_no)
self.outbox.put("reset_election_timeout")
self.request_votes()
@only(State.CANDIDATE)
def request_votes(self):
followers = [
server for server in range(self.num_servers)
if server != self.server_no
]
for server_no in followers:
self._send(
server_no,
RequestVote(
term=self.term,
candidate_log_len=len(self.log),
last_log_term=self.log.last_term,
),
)
# @only([State.CANDIDATE, State.FOLLOWER], silent=True)
def handle_election_timeout(self):
if self.state == State.LEADER:
pass
self.become_candidate()
@only(State.LEADER, silent=True)
def handle_heartbeat(self):
self.outbox.put("reset_election_timeout")
followers = [
server for server in range(self.num_servers)
if server != self.server_no
]
for server_no in followers:
append_entries_msg = self._append_entries_msg(server_no)
if append_entries_msg:
self._send(server_no, self._append_entries_msg(server_no))
@only(State.LEADER)
def leader_log_append(self, entry):
self.log.append(len(self.log),
prev_log_term=self.log.last_term,
entry=LogEntry(self.term, entry))
self.match_index[self.server_no] = len(self.log)
def _handle_command(self, client_id, operation):
# command is any of SetValue, GetValue, DelValue, NoOp
if self.state != State.LEADER:
self._send(client_id, NotTheLeader(self.leader_id))
return
if isinstance(operation, NoOp):
self._send(client_id, Result('ok'))
return
if isinstance(operation, GetValue):
result = self._state_machine.apply(operation)
self._send(client_id, Result(result))
return
# Idempotency check
for log_index, log_entry in enumerate(self.log):
if log_entry.msg.request_id == operation.request_id:
if log_index <= self.last_applied:
self._send(client_id, Result('ok'))
else:
self.unhandled_client_requests[operation] = client_id
# Operation was already in the servers log, don't append it again
return
if isinstance(operation, (SetValue, DelValue)):
self.leader_log_append(operation)
self.unhandled_client_requests[operation] = client_id
return
raise ValueError(
f"Expected command to be in NoOp, GetValue, SetValue, DelValue, got {type(operation)}"
)
def handle_message(self, message: Message):
message_handlers = {
AppendEntries: self._handle_append_entries,
AppendEntriesSucceeded: self._handle_append_entries_succeeded,
AppendEntriesFailed: self._handle_append_entries_failed,
RequestVote: self._handle_request_vote,
VoteGranted: self._handle_vote_granted,
VoteDenied: self._handle_vote_denied,
InvalidTerm: lambda *args, **kwargs: None,
Command: self._handle_command,
}
self._logger.info(
f"Received {message.content} message from server {message.sender}")
if message.term is not None and message.term < self.term:
self._logger.info(
f"Server {message.sender} has a lower term, ignoring")
self._send(message.sender, InvalidTerm())
return
if message.term is not None and message.term > self.term:
self._logger.info(
f"Server {message.sender} has an higher term, updating mine and "
f"converting to follower")
self.become_follower()
self.term = message.term
try:
handler = message_handlers[type(message.content)]
except AttributeError:
raise ValueError(
f"unknown message. expected {message_handlers.keys()}, got {type(message)}"
)
response = handler(message.sender, **message.content._asdict())
if response is not None:
self._send(message.sender, response)
@only(State.CANDIDATE)
def _handle_vote_denied(self, server_no, reason):
self._logger.info(
f"did not get vote from server {server_no} because {reason}")
@only(State.CANDIDATE)
def _handle_vote_granted(self, voter_id):
self.received_votes.add(voter_id)
if len(self.received_votes) > self.num_servers // 2:
self.become_leader()
def _handle_request_vote(self, candidate_id, term, candidate_log_len,
last_log_term):
if term < self.term:
return VoteDenied(
f"Vote came from server on term {term} while own term was {self.term}"
)
if not (self.voted_for is None or self.voted_for == candidate_id):
return VoteDenied(
f"already voted for other candidate: {self.voted_for}")
if candidate_log_len < len(self.log):
return VoteDenied(
f"candidate log was size {candidate_log_len} while own log length was {len(self.log)}"
)
if last_log_term < self.log.last_term:
return VoteDenied(
f"candidate log was on term {last_log_term} while own log length was {self.log.last_term}"
)
self.outbox.put("reset_election_timeout")
self.voted_for = candidate_id
return VoteGranted()
@only(State.LEADER)
def _append_entries_msg(self, server_no):
log_index = self.next_index[server_no]
if log_index >= len(self.log):
self._logger.debug(
f"{server_no} already has all log entries {log_index}/{len(self.log)}"
)
return AppendEntries(
log_index=log_index,
prev_log_term=self.log.last_term,
entry=None,
leader_commit=self.commit_index,
)
self._logger.info(
f"sending AppendEntries RPC to {server_no} for index {log_index}/{len(self.log)}"
)
return AppendEntries(
log_index=log_index,
prev_log_term=self.log[log_index - 1].term,
entry=self.log[log_index],
leader_commit=self.commit_index,
)
def _handle_append_entries(self, leader_id, log_index, prev_log_term,
entry, leader_commit):
"""
Args:
leader_id: so follower can redirect clients
log_index: index in the log where to append to
prev_log_term: according to the leader, the term of the last log entry
entry: the entry to add
leader_commit: the leaders' commit_index
Returns:
"""
if self.state != State.FOLLOWER:
self._logger.info(
f"received append entries call while current state was {self.state}"
)
self.become_follower()
self.outbox.put("reset_election_timeout")
self.leader_id = leader_id
try:
replicated_index = self.log.append(log_index=log_index,
prev_log_term=prev_log_term,
entry=entry)
except (LogNotCaughtUpError, LogDifferentTermError) as e:
return AppendEntriesFailed(reason=e)
# The min in the final step (5) of AppendEntries is necessary, and it needs to be computed with the index of the
# last new entry. It is not sufficient to simply have the function that applies things from your log between
# lastApplied and commitIndex stop when it reaches the end of your log. This is because you may have entries in
# your log that differ from the leader’s log after the entries that the leader sent you (which all match the
# ones in your log). Because #3 dictates that you only truncate your log if you have conflicting entries,
# those won’t be removed, and if leaderCommit is beyond the entries the leader sent you,
# you may apply incorrect entries.
if leader_commit > self.commit_index:
self.commit_index = min(leader_commit, replicated_index)
return AppendEntriesSucceeded(replicated_index)
@only(State.LEADER)
def _handle_append_entries_succeeded(self, other_server_no,
replicated_index):
self.match_index[other_server_no] = replicated_index
self.next_index[other_server_no] = replicated_index + 1
self._update_committed_entries()
def _update_committed_entries(self):
self._logger.info(f"checking whether log entries can be committed")
self._logger.info(f"match index: {self.match_index}")
majority_match_n = sorted(self.match_index)[self.num_servers // 2]
possible_ns = range(self.commit_index,
min(majority_match_n, len(self.log)) + 1)
self._logger.info(
f"possible entries that can be committed: {possible_ns}")
n = None
for possible_n in possible_ns:
if self.log[possible_n].term == self.term:
n = possible_n
if n is not None:
self.commit_index = n
@only(State.LEADER)
def _handle_append_entries_failed(self, other_server_no, reason):
new_try_log_index = self.next_index[other_server_no] - 1
self._logger.info(
f"Received an AppendEntriesFailed message from {other_server_no}. Reason was: {reason} "
f"retrying with log index {new_try_log_index}")
self.next_index[other_server_no] = new_try_log_index
return self._append_entries_msg(server_no=other_server_no)
def write(self):
if not self.persist:
return
with open(self.server_state_path, 'wb') as persisted_state_file:
pickle.dump(
{
'_term': self.term,
'voted_for': self.voted_for,
'log': self.log,
'state': self.state
}, persisted_state_file)
def recover(self):
if not self.persist:
return
try:
with open(self.server_state_path, 'rb') as persisted_state_file:
state = pickle.load(persisted_state_file)
self._term = state['_term']
self.voted_for = state['voted_for']
self.log = state['log']
except FileNotFoundError:
pass
def _send(self, to, content):
self.write()
self.outbox.put(
Message(sender=self.server_no,
term=self.term,
recipient=to,
content=content))