def raft_send_appendentries(raft, udata, node, msg):
server = ffi.from_handle(udata)
assert node
dst_server = ffi.from_handle(lib.raft_node_get_udata(node))
server.network.enqueue_msg(msg, server, dst_server)
# Collect statistics
if server.network.max_entries_in_ae < msg.n_entries:
server.network.max_entries_in_ae = msg.n_entries
return 0
def raft_log(raft, node, udata, buf):
server = ffi.from_handle(lib.raft_get_udata(raft))
# print(server.id, ffi.string(buf).decode('utf8'))
if node != ffi.NULL:
node = ffi.from_handle(lib.raft_node_get_udata(node))
# if server.id in [1] or (node and node.id in [1]):
logging.info('{0}> {1}:{2}: {3}'.format(
server.network.iteration,
server.id,
node.id if node else '',
ffi.string(buf).decode('utf8'),
))
def entry_pop(self, ety, ety_idx):
# logging.warning("POP {} {}".format(self, ety_idx))
e = self._check_committed_entry_popping(ety_idx)
if e != 0:
return e
self.fsm_log.pop()
self.network.log_pops += 1
change = ffi.from_handle(ety.data.buf)
if ety.type == lib.RAFT_LOGTYPE_DEMOTE_NODE:
pass
elif ety.type == lib.RAFT_LOGTYPE_REMOVE_NODE:
if change.node_id == lib.raft_get_nodeid(self.raft):
self.set_connection_status(NODE_CONNECTED)
elif ety.type == lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE:
if change.node_id == lib.raft_get_nodeid(self.raft):
logging.error("POP disconnect {} {}".format(self, ety_idx))
self.set_connection_status(NODE_DISCONNECTED)
elif ety.type == lib.RAFT_LOGTYPE_ADD_NODE:
if change.node_id == lib.raft_get_nodeid(self.raft):
self.set_connection_status(NODE_CONNECTING)
return 0
def periodic(self):
if self.random.randint(1, 100) < self.member_rate:
self.toggle_membership()
if self.random.randint(1, 100) < self.partition_rate:
self.add_partition()
if self.partitions and self.random.randint(1, 100) < self.partition_rate:
self.remove_partition()
if self.random.randint(1, 100) < self.client_rate:
self.push_set_entry(self.random.randint(1, 10), self.random.randint(1, 10))
for server in self.active_servers:
if self.no_random_period:
server.periodic(100)
else:
server.periodic(self.random.randint(1, 100))
# Deadlock detection
if self.latest_applied_log_idx != 0 and self.latest_applied_log_iteration + 5000 < self.iteration:
logging.error("deadlock detected iteration:{0} appliedidx:{1}\n".format(
self.latest_applied_log_iteration,
self.latest_applied_log_idx,
))
sys.exit(1)
# Count leadership changes
leader_node = lib.raft_get_current_leader_node(self.active_servers[0].raft)
if leader_node:
leader = ffi.from_handle(lib.raft_node_get_udata(leader_node))
if self.leader is not leader:
self.leadership_changes += 1
self.leader = leader
def send_snapshot(self, node):
assert not lib.raft_snapshot_is_in_progress(self.raft)
node_sv = ffi.from_handle(lib.raft_node_get_udata(node))
# TODO: Why would this happen?
# seems odd that we would send something to a node that didn't exist
if not node_sv:
return 0
# NOTE:
# In a real server we would have to send the snapshot file to the
# other node. Here we have the convenience of the transfer being
# "immediate".
node_sv.load_snapshot(self.snapshot, self)
return 0
def entry_apply(self, ety, idx):
# collect stats
if self.network.latest_applied_log_idx < idx:
self.network.latest_applied_log_idx = idx
self.network.latest_applied_log_iteration = self.network.iteration
e = self._check_log_matching(ety, idx)
if e is not None:
return e
change = ffi.from_handle(ety.data.buf)
if ety.type == lib.RAFT_LOGTYPE_NORMAL:
self.fsm_dict[change.key] = change.val
elif ety.type == lib.RAFT_LOGTYPE_DEMOTE_NODE:
if change.node_id == lib.raft_get_nodeid(self.raft):
# logging.warning("{} shutting down because of demotion".format(self))
return lib.RAFT_ERR_SHUTDOWN
# Follow up by removing the node by receiving new entry
elif lib.raft_is_leader(self.raft):
new_ety = ffi.new('msg_entry_t*')
new_ety.term = 0
new_ety.id = self.network.new_entry_id()
new_ety.type = lib.RAFT_LOGTYPE_REMOVE_NODE
new_ety.data.buf = ety.data.buf
new_ety.data.len = ffi.sizeof(ety.data.buf)
assert(lib.raft_entry_is_cfg_change(new_ety))
e = self.recv_entry(new_ety)
assert e == 0
elif ety.type == lib.RAFT_LOGTYPE_REMOVE_NODE:
if change.node_id == lib.raft_get_nodeid(self.raft):
# logging.warning("{} shutting down because of removal".format(self))
return lib.RAFT_ERR_SHUTDOWN
elif ety.type == lib.RAFT_LOGTYPE_ADD_NODE:
if change.node_id == self.id:
self.set_connection_status(NODE_CONNECTED)
elif ety.type == lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE:
pass
return 0
def raft_notify_membership_event(raft, udata, node, event_type):
return ffi.from_handle(udata).notify_membership_event(node, event_type)
def raft_node_has_sufficient_logs(raft, udata, node):
return ffi.from_handle(udata).node_has_sufficient_entries(node)
def raft_logentry_get_node_id(raft, udata, ety, ety_idx):
change_entry = ffi.from_handle(ety.data.buf)
assert isinstance(change_entry, ChangeRaftEntry)
return change_entry.node_id
def raft_logentry_pop(raft, udata, ety, ety_idx):
return ffi.from_handle(udata).entry_pop(ety, ety_idx)
def raft_logentry_offer(raft, udata, ety, ety_idx):
return ffi.from_handle(udata).entry_append(ety, ety_idx)
def raft_persist_term(raft, udata, term, vote):
return ffi.from_handle(udata).persist_term(term, vote)
def raft_persist_vote(raft, udata, voted_for):
return ffi.from_handle(udata).persist_vote(voted_for)
def raft_applylog(raft, udata, ety, idx):
try:
return ffi.from_handle(udata).entry_apply(ety, idx)
except:
return lib.RAFT_ERR_SHUTDOWN
def raft_send_snapshot(raft, udata, node):
return ffi.from_handle(udata).send_snapshot(node)
def raft_send_requestvote(raft, udata, node, msg):
server = ffi.from_handle(udata)
dst_server = ffi.from_handle(lib.raft_node_get_udata(node))
server.network.enqueue_msg(msg, server, dst_server)
return 0
