def debug_statistics(self):
return {
"node":
lib.raft_get_nodeid(self.raft),
"state":
state2str(lib.raft_get_state(self.raft)),
"current_idx":
lib.raft_get_current_idx(self.raft),
"last_log_term":
lib.raft_get_last_log_term(self.raft),
"current_term":
lib.raft_get_current_term(self.raft),
"commit_idx":
lib.raft_get_commit_idx(self.raft),
"last_applied_idx":
lib.raft_get_last_applied_idx(self.raft),
"log_count":
lib.raft_get_log_count(self.raft),
"peers":
lib.raft_get_num_nodes(self.raft),
"voting_peers":
lib.raft_get_num_voting_nodes(self.raft),
"connection_status":
connectstatus2str(self.connection_status),
"voting_change_in_progress":
lib.raft_voting_change_is_in_progress(self.raft),
"removed":
getattr(self, 'removed', False),
}
def remove_member(self):
if not self.leader:
logging.error('no leader')
return
leader = self.leader
server = self.random.choice(self.active_servers)
if not lib.raft_is_leader(leader.raft):
return
if lib.raft_voting_change_is_in_progress(leader.raft):
# logging.error('{} voting change in progress'.format(server))
return
if leader == server:
# logging.error('can not remove leader')
return
if server.connection_status in [NODE_CONNECTING, NODE_DISCONNECTING]:
# logging.error('can not remove server that is changing connection status')
return
if NODE_DISCONNECTED == server.connection_status:
self.remove_server(server)
return
# Create a new configuration entry to be processed by the leader
ety = ffi.new('msg_entry_t*')
ety.term = 0
ety.id = self.new_entry_id()
assert server.connection_status == NODE_CONNECTED
ety.type = lib.RAFT_LOGTYPE_DEMOTE_NODE
change = ffi.new_handle(ChangeRaftEntry(server.id))
ety.data.buf = change
ety.data.len = ffi.sizeof(ety.data.buf)
assert (lib.raft_entry_is_cfg_change(ety))
self.entries.append((ety, change))
e = leader.recv_entry(ety)
if 0 != e:
logging.error(err2str(e))
return
else:
self.num_membership_changes += 1
# Wake up new node
assert NODE_CONNECTED == server.connection_status
server.set_connection_status(NODE_DISCONNECTING)
def add_member(self):
if net.num_of_servers <= len(self.active_servers):
return
if not self.leader:
logging.error('no leader')
return
leader = self.leader
if not lib.raft_is_leader(leader.raft):
return
if lib.raft_voting_change_is_in_progress(leader.raft):
# logging.error('{} voting change in progress'.format(server))
return
server = RaftServer(self)
# Create a new configuration entry to be processed by the leader
ety = ffi.new('msg_entry_t*')
ety.term = 0
ety.id = self.new_entry_id()
ety.type = lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE
change = ffi.new_handle(ChangeRaftEntry(server.id))
ety.data.buf = change
ety.data.len = ffi.sizeof(ety.data.buf)
assert (lib.raft_entry_is_cfg_change(ety))
self.entries.append((ety, change))
e = leader.recv_entry(ety)
if 0 != e:
logging.error(err2str(e))
return
else:
self.num_membership_changes += 1
# Wake up new node
server.set_connection_status(NODE_CONNECTING)
assert server.udata
added_node = lib.raft_add_non_voting_node(server.raft, server.udata,
server.id, 1)
assert added_node
def remove_member(self):
if not self.leader:
logger.error('no leader')
return
leader = self.leader
server = self.random.choice(self.active_servers)
if not lib.raft_is_leader(leader.raft):
return
if lib.raft_voting_change_is_in_progress(leader.raft):
# logger.error('{} voting change in progress'.format(server))
return
if leader == server:
# logger.error('can not remove leader')
return
if server.connection_status in [NODE_CONNECTING, NODE_DISCONNECTING]:
# logger.error('can not remove server that is changing connection status')
return
if NODE_DISCONNECTED == server.connection_status:
self.remove_server(server)
return
# Create a new configuration entry to be processed by the leader
ety = self.add_entry(lib.RAFT_LOGTYPE_DEMOTE_NODE,
ChangeRaftEntry(server.id))
assert server.connection_status == NODE_CONNECTED
assert (lib.raft_entry_is_cfg_change(ety))
e = leader.recv_entry(ety)
if 0 != e:
logger.error(err2str(e))
return
else:
self.num_membership_changes += 1
# Wake up new node
assert NODE_CONNECTED == server.connection_status
server.set_connection_status(NODE_DISCONNECTING)
def add_member(self):
if net.num_of_servers <= len(self.active_servers):
return
if not self.leader:
logger.error('no leader')
return
leader = self.leader
if not lib.raft_is_leader(leader.raft):
return
if lib.raft_voting_change_is_in_progress(leader.raft):
# logger.error('{} voting change in progress'.format(server))
return
server = RaftServer(self)
# Create a new configuration entry to be processed by the leader
ety = self.add_entry(lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE,
ChangeRaftEntry(server.id))
assert (lib.raft_entry_is_cfg_change(ety))
e = leader.recv_entry(ety)
if 0 != e:
logger.error(err2str(e))
return
else:
self.num_membership_changes += 1
# Wake up new node
server.set_connection_status(NODE_CONNECTING)
assert server.udata
added_node = lib.raft_add_non_voting_node(server.raft, server.udata,
server.id, 1)
assert added_node
def toggle_membership(self):
if not self.leader:
logging.error('no leader')
return
leader = self.leader
server = self.random.choice(self.active_servers)
if not lib.raft_is_leader(leader.raft):
return
if lib.raft_voting_change_is_in_progress(leader.raft):
# logging.error('{} voting change in progress'.format(server))
return
if leader == server:
# logging.error('can not remove leader')
return
if server.connection_status in [NODE_CONNECTING, NODE_DISCONNECTING]:
# logging.error('can not remove server that is changing connection status')
return
if NODE_DISCONNECTED == server.connection_status:
self.remove_server(server)
server = RaftServer(self)
# Create a new configuration entry to be processed by the leader
ety = ffi.new('msg_entry_t*')
ety.term = 0
ety.id = self.new_entry_id()
if server.connection_status == NODE_CONNECTED:
ety.type = lib.RAFT_LOGTYPE_DEMOTE_NODE
else:
ety.type = lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE
change = ffi.new_handle(ChangeRaftEntry(server.id))
ety.data.buf = change
ety.data.len = ffi.sizeof(ety.data.buf)
assert(lib.raft_entry_is_cfg_change(ety))
self.entries.append((ety, change))
# logging.warning("{}> {} Membership change {}: ety.id:{} id:{} {}".format(
# self.iteration,
# server,
# "ADD" if ety.type == lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE else "DEMOTE",
# ety.id,
# lib.raft_get_nodeid(leader.raft),
# server.id,
# ))
e = leader.recv_entry(ety)
if 0 != e:
logging.error(err2str(e))
return
else:
self.num_membership_changes += 1
# Wake up new node
if NODE_DISCONNECTED == server.connection_status:
server.set_connection_status(NODE_CONNECTING)
assert server.udata
added_node = lib.raft_add_non_voting_node(server.raft, server.udata, server.id, 1)
assert added_node
elif NODE_CONNECTED == server.connection_status:
server.set_connection_status(NODE_DISCONNECTING)
else:
logging.warning('Unknown status set {}'.format(server.connection_status))