def raft_send_appendentries(raft, udata, node, msg): server = ffi.from_handle(udata) assert node dst_server = ffi.from_handle(lib.raft_node_get_udata(node)) server.network.enqueue_msg(msg, server, dst_server) # Collect statistics if server.network.max_entries_in_ae < msg.n_entries: server.network.max_entries_in_ae = msg.n_entries return 0
def raft_log(raft, node, udata, buf): server = ffi.from_handle(lib.raft_get_udata(raft)) # print(server.id, ffi.string(buf).decode('utf8')) if node != ffi.NULL: node = ffi.from_handle(lib.raft_node_get_udata(node)) # if server.id in [1] or (node and node.id in [1]): logging.info('{0}> {1}:{2}: {3}'.format( server.network.iteration, server.id, node.id if node else '', ffi.string(buf).decode('utf8'), ))
def entry_pop(self, ety, ety_idx): # logging.warning("POP {} {}".format(self, ety_idx)) e = self._check_committed_entry_popping(ety_idx) if e != 0: return e self.fsm_log.pop() self.network.log_pops += 1 change = ffi.from_handle(ety.data.buf) if ety.type == lib.RAFT_LOGTYPE_DEMOTE_NODE: pass elif ety.type == lib.RAFT_LOGTYPE_REMOVE_NODE: if change.node_id == lib.raft_get_nodeid(self.raft): self.set_connection_status(NODE_CONNECTED) elif ety.type == lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE: if change.node_id == lib.raft_get_nodeid(self.raft): logging.error("POP disconnect {} {}".format(self, ety_idx)) self.set_connection_status(NODE_DISCONNECTED) elif ety.type == lib.RAFT_LOGTYPE_ADD_NODE: if change.node_id == lib.raft_get_nodeid(self.raft): self.set_connection_status(NODE_CONNECTING) return 0
def periodic(self): if self.random.randint(1, 100) < self.member_rate: self.toggle_membership() if self.random.randint(1, 100) < self.partition_rate: self.add_partition() if self.partitions and self.random.randint(1, 100) < self.partition_rate: self.remove_partition() if self.random.randint(1, 100) < self.client_rate: self.push_set_entry(self.random.randint(1, 10), self.random.randint(1, 10)) for server in self.active_servers: if self.no_random_period: server.periodic(100) else: server.periodic(self.random.randint(1, 100)) # Deadlock detection if self.latest_applied_log_idx != 0 and self.latest_applied_log_iteration + 5000 < self.iteration: logging.error("deadlock detected iteration:{0} appliedidx:{1}\n".format( self.latest_applied_log_iteration, self.latest_applied_log_idx, )) sys.exit(1) # Count leadership changes leader_node = lib.raft_get_current_leader_node(self.active_servers[0].raft) if leader_node: leader = ffi.from_handle(lib.raft_node_get_udata(leader_node)) if self.leader is not leader: self.leadership_changes += 1 self.leader = leader
def send_snapshot(self, node): assert not lib.raft_snapshot_is_in_progress(self.raft) node_sv = ffi.from_handle(lib.raft_node_get_udata(node)) # TODO: Why would this happen? # seems odd that we would send something to a node that didn't exist if not node_sv: return 0 # NOTE: # In a real server we would have to send the snapshot file to the # other node. Here we have the convenience of the transfer being # "immediate". node_sv.load_snapshot(self.snapshot, self) return 0
def entry_apply(self, ety, idx): # collect stats if self.network.latest_applied_log_idx < idx: self.network.latest_applied_log_idx = idx self.network.latest_applied_log_iteration = self.network.iteration e = self._check_log_matching(ety, idx) if e is not None: return e change = ffi.from_handle(ety.data.buf) if ety.type == lib.RAFT_LOGTYPE_NORMAL: self.fsm_dict[change.key] = change.val elif ety.type == lib.RAFT_LOGTYPE_DEMOTE_NODE: if change.node_id == lib.raft_get_nodeid(self.raft): # logging.warning("{} shutting down because of demotion".format(self)) return lib.RAFT_ERR_SHUTDOWN # Follow up by removing the node by receiving new entry elif lib.raft_is_leader(self.raft): new_ety = ffi.new('msg_entry_t*') new_ety.term = 0 new_ety.id = self.network.new_entry_id() new_ety.type = lib.RAFT_LOGTYPE_REMOVE_NODE new_ety.data.buf = ety.data.buf new_ety.data.len = ffi.sizeof(ety.data.buf) assert(lib.raft_entry_is_cfg_change(new_ety)) e = self.recv_entry(new_ety) assert e == 0 elif ety.type == lib.RAFT_LOGTYPE_REMOVE_NODE: if change.node_id == lib.raft_get_nodeid(self.raft): # logging.warning("{} shutting down because of removal".format(self)) return lib.RAFT_ERR_SHUTDOWN elif ety.type == lib.RAFT_LOGTYPE_ADD_NODE: if change.node_id == self.id: self.set_connection_status(NODE_CONNECTED) elif ety.type == lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE: pass return 0
def raft_notify_membership_event(raft, udata, node, event_type): return ffi.from_handle(udata).notify_membership_event(node, event_type)
def raft_node_has_sufficient_logs(raft, udata, node): return ffi.from_handle(udata).node_has_sufficient_entries(node)
def raft_logentry_get_node_id(raft, udata, ety, ety_idx): change_entry = ffi.from_handle(ety.data.buf) assert isinstance(change_entry, ChangeRaftEntry) return change_entry.node_id
def raft_logentry_pop(raft, udata, ety, ety_idx): return ffi.from_handle(udata).entry_pop(ety, ety_idx)
def raft_logentry_offer(raft, udata, ety, ety_idx): return ffi.from_handle(udata).entry_append(ety, ety_idx)
def raft_persist_term(raft, udata, term, vote): return ffi.from_handle(udata).persist_term(term, vote)
def raft_persist_vote(raft, udata, voted_for): return ffi.from_handle(udata).persist_vote(voted_for)
def raft_applylog(raft, udata, ety, idx): try: return ffi.from_handle(udata).entry_apply(ety, idx) except: return lib.RAFT_ERR_SHUTDOWN
def raft_send_snapshot(raft, udata, node): return ffi.from_handle(udata).send_snapshot(node)
def raft_send_requestvote(raft, udata, node, msg): server = ffi.from_handle(udata) dst_server = ffi.from_handle(lib.raft_node_get_udata(node)) server.network.enqueue_msg(msg, server, dst_server) return 0