def poll_message(self, msg): msg_type = ffi.getctype(ffi.typeof(msg.data)) if msg_type == 'msg_appendentries_t *': node = lib.raft_get_node(msg.sendee.raft, msg.sendor.id) response = ffi.new('msg_appendentries_response_t*') e = lib.raft_recv_appendentries(msg.sendee.raft, node, msg.data, response) if lib.RAFT_ERR_SHUTDOWN == e: logging.error('Catastrophic') print(msg.sendee.debug_log()) print(msg.sendor.debug_log()) sys.exit(1) elif lib.RAFT_ERR_NEEDS_SNAPSHOT == e: pass # TODO: pretend as if snapshot works else: self.enqueue_msg(response, msg.sendee, msg.sendor) elif msg_type == 'msg_appendentries_response_t *': node = lib.raft_get_node(msg.sendee.raft, msg.sendor.id) lib.raft_recv_appendentries_response(msg.sendee.raft, node, msg.data) elif msg_type == 'msg_requestvote_t *': response = ffi.new('msg_requestvote_response_t*') node = lib.raft_get_node(msg.sendee.raft, msg.sendor.id) lib.raft_recv_requestvote(msg.sendee.raft, node, msg.data, response) self.enqueue_msg(response, msg.sendee, msg.sendor) elif msg_type == 'msg_requestvote_response_t *': node = lib.raft_get_node(msg.sendee.raft, msg.sendor.id) e = lib.raft_recv_requestvote_response(msg.sendee.raft, node, msg.data) if lib.RAFT_ERR_SHUTDOWN == e: msg.sendor.shutdown() else: assert False
def __init__(self, network): self.connection_status = NODE_DISCONNECTED self.raft = lib.raft_new() self.udata = ffi.new_handle(self) network.add_server(self) self.load_callbacks() cbs = ffi.new('raft_cbs_t*') cbs.send_requestvote = self.raft_send_requestvote cbs.send_appendentries = self.raft_send_appendentries cbs.send_snapshot = self.raft_send_snapshot cbs.applylog = self.raft_applylog cbs.persist_vote = self.raft_persist_vote cbs.persist_term = self.raft_persist_term cbs.log_get_node_id = self.raft_logentry_get_node_id cbs.node_has_sufficient_logs = self.raft_node_has_sufficient_logs cbs.notify_membership_event = self.raft_notify_membership_event cbs.log = self.raft_log log_cbs = ffi.new('raft_log_cbs_t*') log_cbs.log_offer = self.raft_logentry_offer log_cbs.log_poll = self.raft_logentry_poll log_cbs.log_pop = self.raft_logentry_pop lib.raft_set_callbacks(self.raft, cbs, self.udata) lib.log_set_callbacks(lib.raft_get_log(self.raft), log_cbs, self.raft) lib.raft_set_election_timeout(self.raft, 500) self.fsm_dict = {} self.fsm_log = []
def prep_dynamic_configuration(self): """ Add configuration change for leader's node """ server = self.active_servers[0] self.leader = server server.set_connection_status(NODE_CONNECTED) lib.raft_add_non_voting_node(server.raft, server.udata, server.id, 1) lib.raft_become_leader(server.raft) # Configuration change entry to bootstrap other nodes ety = ffi.new('msg_entry_t*') ety.term = 0 ety.type = lib.RAFT_LOGTYPE_ADD_NODE ety.id = self.new_entry_id() change = ffi.new_handle(ChangeRaftEntry(server.id)) ety.data.buf = change ety.data.len = ffi.sizeof(ety.data.buf) self.entries.append((ety, change)) e = server.recv_entry(ety) assert e == 0 lib.raft_set_commit_idx(server.raft, 1) e = lib.raft_apply_all(server.raft) assert e == 0
def push_set_entry(self, k, v): for sv in self.active_servers: if lib.raft_is_leader(sv.raft): ety = ffi.new('msg_entry_t*') ety.term = 0 ety.id = self.new_entry_id() ety.type = lib.RAFT_LOGTYPE_NORMAL change = ffi.new_handle(SetRaftEntry(k, v)) ety.data.buf = change ety.data.len = ffi.sizeof(ety.data.buf) self.entries.append((ety, change)) e = sv.recv_entry(ety) assert e == 0 break
def remove_member(self): if not self.leader: logging.error('no leader') return leader = self.leader server = self.random.choice(self.active_servers) if not lib.raft_is_leader(leader.raft): return if lib.raft_voting_change_is_in_progress(leader.raft): # logging.error('{} voting change in progress'.format(server)) return if leader == server: # logging.error('can not remove leader') return if server.connection_status in [NODE_CONNECTING, NODE_DISCONNECTING]: # logging.error('can not remove server that is changing connection status') return if NODE_DISCONNECTED == server.connection_status: self.remove_server(server) return # Create a new configuration entry to be processed by the leader ety = ffi.new('msg_entry_t*') ety.term = 0 ety.id = self.new_entry_id() assert server.connection_status == NODE_CONNECTED ety.type = lib.RAFT_LOGTYPE_DEMOTE_NODE change = ffi.new_handle(ChangeRaftEntry(server.id)) ety.data.buf = change ety.data.len = ffi.sizeof(ety.data.buf) assert (lib.raft_entry_is_cfg_change(ety)) self.entries.append((ety, change)) e = leader.recv_entry(ety) if 0 != e: logging.error(err2str(e)) return else: self.num_membership_changes += 1 # Wake up new node assert NODE_CONNECTED == server.connection_status server.set_connection_status(NODE_DISCONNECTING)
def node_has_sufficient_entries(self, node): assert(not lib.raft_node_is_voting(node)) ety = ffi.new('msg_entry_t*') ety.term = 0 ety.id = self.network.new_entry_id() ety.type = lib.RAFT_LOGTYPE_ADD_NODE change = ffi.new_handle(ChangeRaftEntry(lib.raft_node_get_id(node))) ety.data.buf = change ety.data.len = ffi.sizeof(ety.data.buf) self.network.entries.append((ety, change)) assert(lib.raft_entry_is_cfg_change(ety)) # FIXME: leak e = self.recv_entry(ety) # print(err2str(e)) assert e == 0 return 0
def entry_apply(self, ety, idx): # collect stats if self.network.latest_applied_log_idx < idx: self.network.latest_applied_log_idx = idx self.network.latest_applied_log_iteration = self.network.iteration e = self._check_log_matching(ety, idx) if e is not None: return e change = ffi.from_handle(ety.data.buf) if ety.type == lib.RAFT_LOGTYPE_NORMAL: self.fsm_dict[change.key] = change.val elif ety.type == lib.RAFT_LOGTYPE_DEMOTE_NODE: if change.node_id == lib.raft_get_nodeid(self.raft): # logging.warning("{} shutting down because of demotion".format(self)) return lib.RAFT_ERR_SHUTDOWN # Follow up by removing the node by receiving new entry elif lib.raft_is_leader(self.raft): new_ety = ffi.new('msg_entry_t*') new_ety.term = 0 new_ety.id = self.network.new_entry_id() new_ety.type = lib.RAFT_LOGTYPE_REMOVE_NODE new_ety.data.buf = ety.data.buf new_ety.data.len = ffi.sizeof(ety.data.buf) assert(lib.raft_entry_is_cfg_change(new_ety)) e = self.recv_entry(new_ety) assert e == 0 elif ety.type == lib.RAFT_LOGTYPE_REMOVE_NODE: if change.node_id == lib.raft_get_nodeid(self.raft): # logging.warning("{} shutting down because of removal".format(self)) return lib.RAFT_ERR_SHUTDOWN elif ety.type == lib.RAFT_LOGTYPE_ADD_NODE: if change.node_id == self.id: self.set_connection_status(NODE_CONNECTED) elif ety.type == lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE: pass return 0
def add_member(self): if net.num_of_servers <= len(self.active_servers): return if not self.leader: logging.error('no leader') return leader = self.leader if not lib.raft_is_leader(leader.raft): return if lib.raft_voting_change_is_in_progress(leader.raft): # logging.error('{} voting change in progress'.format(server)) return server = RaftServer(self) # Create a new configuration entry to be processed by the leader ety = ffi.new('msg_entry_t*') ety.term = 0 ety.id = self.new_entry_id() ety.type = lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE change = ffi.new_handle(ChangeRaftEntry(server.id)) ety.data.buf = change ety.data.len = ffi.sizeof(ety.data.buf) assert (lib.raft_entry_is_cfg_change(ety)) self.entries.append((ety, change)) e = leader.recv_entry(ety) if 0 != e: logging.error(err2str(e)) return else: self.num_membership_changes += 1 # Wake up new node server.set_connection_status(NODE_CONNECTING) assert server.udata added_node = lib.raft_add_non_voting_node(server.raft, server.udata, server.id, 1) assert added_node
def load_snapshot(self, snapshot, other): # logging.warning('{} loading snapshot'.format(self)) e = lib.raft_begin_load_snapshot( self.raft, snapshot.last_term, snapshot.last_idx, ) if e == -1: return 0 elif e == lib.RAFT_ERR_SNAPSHOT_ALREADY_LOADED: return 0 elif e == 0: pass else: assert False # Send appendentries response for this snapshot response = ffi.new('msg_appendentries_response_t*') response.success = 1 response.current_idx = snapshot.last_idx response.term = lib.raft_get_current_term(self.raft) response.first_idx = response.current_idx self.network.enqueue_msg(response, self, other) node_id = lib.raft_get_nodeid(self.raft) # set membership configuration according to snapshot for member in snapshot.members: if -1 == member.id: continue node = lib.raft_get_node(self.raft, member.id) if not node: udata = ffi.NULL try: node_sv = self.network.id2server(member.id) udata = node_sv.udata except ServerDoesNotExist: pass node = lib.raft_add_node(self.raft, udata, member.id, member.id == node_id) lib.raft_node_set_active(node, 1) if member.voting and not lib.raft_node_is_voting(node): lib.raft_node_set_voting(node, 1) elif not member.voting and lib.raft_node_is_voting(node): lib.raft_node_set_voting(node, 0) if node_id != member.id: assert node # TODO: this is quite ugly # we should have a function that removes all nodes by ourself # if (!raft_get_my_node(self->raft)) */ # raft_add_non_voting_node(self->raft, NULL, node_id, 1); */ e = lib.raft_end_load_snapshot(self.raft) assert(0 == e) assert(lib.raft_get_log_count(self.raft) == 0) self.do_membership_snapshot() self.snapshot.image = dict(snapshot.image) self.snapshot.last_term = snapshot.last_term self.snapshot.last_idx = snapshot.last_idx assert(lib.raft_get_my_node(self.raft)) # assert(sv->snapshot_fsm); self.fsm_dict = dict(snapshot.image) logging.warning('{} loaded snapshot t:{} idx:{}'.format( self, snapshot.last_term, snapshot.last_idx))
def recv_entry(self, ety): # FIXME: leak response = ffi.new('msg_entry_response_t*') return lib.raft_recv_entry(self.raft, ety, response)
def toggle_membership(self): if not self.leader: logging.error('no leader') return leader = self.leader server = self.random.choice(self.active_servers) if not lib.raft_is_leader(leader.raft): return if lib.raft_voting_change_is_in_progress(leader.raft): # logging.error('{} voting change in progress'.format(server)) return if leader == server: # logging.error('can not remove leader') return if server.connection_status in [NODE_CONNECTING, NODE_DISCONNECTING]: # logging.error('can not remove server that is changing connection status') return if NODE_DISCONNECTED == server.connection_status: self.remove_server(server) server = RaftServer(self) # Create a new configuration entry to be processed by the leader ety = ffi.new('msg_entry_t*') ety.term = 0 ety.id = self.new_entry_id() if server.connection_status == NODE_CONNECTED: ety.type = lib.RAFT_LOGTYPE_DEMOTE_NODE else: ety.type = lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE change = ffi.new_handle(ChangeRaftEntry(server.id)) ety.data.buf = change ety.data.len = ffi.sizeof(ety.data.buf) assert(lib.raft_entry_is_cfg_change(ety)) self.entries.append((ety, change)) # logging.warning("{}> {} Membership change {}: ety.id:{} id:{} {}".format( # self.iteration, # server, # "ADD" if ety.type == lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE else "DEMOTE", # ety.id, # lib.raft_get_nodeid(leader.raft), # server.id, # )) e = leader.recv_entry(ety) if 0 != e: logging.error(err2str(e)) return else: self.num_membership_changes += 1 # Wake up new node if NODE_DISCONNECTED == server.connection_status: server.set_connection_status(NODE_CONNECTING) assert server.udata added_node = lib.raft_add_non_voting_node(server.raft, server.udata, server.id, 1) assert added_node elif NODE_CONNECTED == server.connection_status: server.set_connection_status(NODE_DISCONNECTING) else: logging.warning('Unknown status set {}'.format(server.connection_status))