def poll_message(self, msg):
msg_type = ffi.getctype(ffi.typeof(msg.data))
if msg_type == 'msg_appendentries_t *':
node = lib.raft_get_node(msg.sendee.raft, msg.sendor.id)
response = ffi.new('msg_appendentries_response_t*')
e = lib.raft_recv_appendentries(msg.sendee.raft, node, msg.data, response)
if lib.RAFT_ERR_SHUTDOWN == e:
logging.error('Catastrophic')
print(msg.sendee.debug_log())
print(msg.sendor.debug_log())
sys.exit(1)
elif lib.RAFT_ERR_NEEDS_SNAPSHOT == e:
pass # TODO: pretend as if snapshot works
else:
self.enqueue_msg(response, msg.sendee, msg.sendor)
elif msg_type == 'msg_appendentries_response_t *':
node = lib.raft_get_node(msg.sendee.raft, msg.sendor.id)
lib.raft_recv_appendentries_response(msg.sendee.raft, node, msg.data)
elif msg_type == 'msg_requestvote_t *':
response = ffi.new('msg_requestvote_response_t*')
node = lib.raft_get_node(msg.sendee.raft, msg.sendor.id)
lib.raft_recv_requestvote(msg.sendee.raft, node, msg.data, response)
self.enqueue_msg(response, msg.sendee, msg.sendor)
elif msg_type == 'msg_requestvote_response_t *':
node = lib.raft_get_node(msg.sendee.raft, msg.sendor.id)
e = lib.raft_recv_requestvote_response(msg.sendee.raft, node, msg.data)
if lib.RAFT_ERR_SHUTDOWN == e:
msg.sendor.shutdown()
else:
assert False
def __init__(self, network):
self.connection_status = NODE_DISCONNECTED
self.raft = lib.raft_new()
self.udata = ffi.new_handle(self)
network.add_server(self)
self.load_callbacks()
cbs = ffi.new('raft_cbs_t*')
cbs.send_requestvote = self.raft_send_requestvote
cbs.send_appendentries = self.raft_send_appendentries
cbs.send_snapshot = self.raft_send_snapshot
cbs.applylog = self.raft_applylog
cbs.persist_vote = self.raft_persist_vote
cbs.persist_term = self.raft_persist_term
cbs.log_get_node_id = self.raft_logentry_get_node_id
cbs.node_has_sufficient_logs = self.raft_node_has_sufficient_logs
cbs.notify_membership_event = self.raft_notify_membership_event
cbs.log = self.raft_log
log_cbs = ffi.new('raft_log_cbs_t*')
log_cbs.log_offer = self.raft_logentry_offer
log_cbs.log_poll = self.raft_logentry_poll
log_cbs.log_pop = self.raft_logentry_pop
lib.raft_set_callbacks(self.raft, cbs, self.udata)
lib.log_set_callbacks(lib.raft_get_log(self.raft), log_cbs, self.raft)
lib.raft_set_election_timeout(self.raft, 500)
self.fsm_dict = {}
self.fsm_log = []
def prep_dynamic_configuration(self):
"""
Add configuration change for leader's node
"""
server = self.active_servers[0]
self.leader = server
server.set_connection_status(NODE_CONNECTED)
lib.raft_add_non_voting_node(server.raft, server.udata, server.id, 1)
lib.raft_become_leader(server.raft)
# Configuration change entry to bootstrap other nodes
ety = ffi.new('msg_entry_t*')
ety.term = 0
ety.type = lib.RAFT_LOGTYPE_ADD_NODE
ety.id = self.new_entry_id()
change = ffi.new_handle(ChangeRaftEntry(server.id))
ety.data.buf = change
ety.data.len = ffi.sizeof(ety.data.buf)
self.entries.append((ety, change))
e = server.recv_entry(ety)
assert e == 0
lib.raft_set_commit_idx(server.raft, 1)
e = lib.raft_apply_all(server.raft)
assert e == 0
def push_set_entry(self, k, v):
for sv in self.active_servers:
if lib.raft_is_leader(sv.raft):
ety = ffi.new('msg_entry_t*')
ety.term = 0
ety.id = self.new_entry_id()
ety.type = lib.RAFT_LOGTYPE_NORMAL
change = ffi.new_handle(SetRaftEntry(k, v))
ety.data.buf = change
ety.data.len = ffi.sizeof(ety.data.buf)
self.entries.append((ety, change))
e = sv.recv_entry(ety)
assert e == 0
break
def remove_member(self):
if not self.leader:
logging.error('no leader')
return
leader = self.leader
server = self.random.choice(self.active_servers)
if not lib.raft_is_leader(leader.raft):
return
if lib.raft_voting_change_is_in_progress(leader.raft):
# logging.error('{} voting change in progress'.format(server))
return
if leader == server:
# logging.error('can not remove leader')
return
if server.connection_status in [NODE_CONNECTING, NODE_DISCONNECTING]:
# logging.error('can not remove server that is changing connection status')
return
if NODE_DISCONNECTED == server.connection_status:
self.remove_server(server)
return
# Create a new configuration entry to be processed by the leader
ety = ffi.new('msg_entry_t*')
ety.term = 0
ety.id = self.new_entry_id()
assert server.connection_status == NODE_CONNECTED
ety.type = lib.RAFT_LOGTYPE_DEMOTE_NODE
change = ffi.new_handle(ChangeRaftEntry(server.id))
ety.data.buf = change
ety.data.len = ffi.sizeof(ety.data.buf)
assert (lib.raft_entry_is_cfg_change(ety))
self.entries.append((ety, change))
e = leader.recv_entry(ety)
if 0 != e:
logging.error(err2str(e))
return
else:
self.num_membership_changes += 1
# Wake up new node
assert NODE_CONNECTED == server.connection_status
server.set_connection_status(NODE_DISCONNECTING)
def node_has_sufficient_entries(self, node):
assert(not lib.raft_node_is_voting(node))
ety = ffi.new('msg_entry_t*')
ety.term = 0
ety.id = self.network.new_entry_id()
ety.type = lib.RAFT_LOGTYPE_ADD_NODE
change = ffi.new_handle(ChangeRaftEntry(lib.raft_node_get_id(node)))
ety.data.buf = change
ety.data.len = ffi.sizeof(ety.data.buf)
self.network.entries.append((ety, change))
assert(lib.raft_entry_is_cfg_change(ety))
# FIXME: leak
e = self.recv_entry(ety)
# print(err2str(e))
assert e == 0
return 0
def entry_apply(self, ety, idx):
# collect stats
if self.network.latest_applied_log_idx < idx:
self.network.latest_applied_log_idx = idx
self.network.latest_applied_log_iteration = self.network.iteration
e = self._check_log_matching(ety, idx)
if e is not None:
return e
change = ffi.from_handle(ety.data.buf)
if ety.type == lib.RAFT_LOGTYPE_NORMAL:
self.fsm_dict[change.key] = change.val
elif ety.type == lib.RAFT_LOGTYPE_DEMOTE_NODE:
if change.node_id == lib.raft_get_nodeid(self.raft):
# logging.warning("{} shutting down because of demotion".format(self))
return lib.RAFT_ERR_SHUTDOWN
# Follow up by removing the node by receiving new entry
elif lib.raft_is_leader(self.raft):
new_ety = ffi.new('msg_entry_t*')
new_ety.term = 0
new_ety.id = self.network.new_entry_id()
new_ety.type = lib.RAFT_LOGTYPE_REMOVE_NODE
new_ety.data.buf = ety.data.buf
new_ety.data.len = ffi.sizeof(ety.data.buf)
assert(lib.raft_entry_is_cfg_change(new_ety))
e = self.recv_entry(new_ety)
assert e == 0
elif ety.type == lib.RAFT_LOGTYPE_REMOVE_NODE:
if change.node_id == lib.raft_get_nodeid(self.raft):
# logging.warning("{} shutting down because of removal".format(self))
return lib.RAFT_ERR_SHUTDOWN
elif ety.type == lib.RAFT_LOGTYPE_ADD_NODE:
if change.node_id == self.id:
self.set_connection_status(NODE_CONNECTED)
elif ety.type == lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE:
pass
return 0
def add_member(self):
if net.num_of_servers <= len(self.active_servers):
return
if not self.leader:
logging.error('no leader')
return
leader = self.leader
if not lib.raft_is_leader(leader.raft):
return
if lib.raft_voting_change_is_in_progress(leader.raft):
# logging.error('{} voting change in progress'.format(server))
return
server = RaftServer(self)
# Create a new configuration entry to be processed by the leader
ety = ffi.new('msg_entry_t*')
ety.term = 0
ety.id = self.new_entry_id()
ety.type = lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE
change = ffi.new_handle(ChangeRaftEntry(server.id))
ety.data.buf = change
ety.data.len = ffi.sizeof(ety.data.buf)
assert (lib.raft_entry_is_cfg_change(ety))
self.entries.append((ety, change))
e = leader.recv_entry(ety)
if 0 != e:
logging.error(err2str(e))
return
else:
self.num_membership_changes += 1
# Wake up new node
server.set_connection_status(NODE_CONNECTING)
assert server.udata
added_node = lib.raft_add_non_voting_node(server.raft, server.udata,
server.id, 1)
assert added_node
def load_snapshot(self, snapshot, other):
# logging.warning('{} loading snapshot'.format(self))
e = lib.raft_begin_load_snapshot(
self.raft,
snapshot.last_term,
snapshot.last_idx,
)
if e == -1:
return 0
elif e == lib.RAFT_ERR_SNAPSHOT_ALREADY_LOADED:
return 0
elif e == 0:
pass
else:
assert False
# Send appendentries response for this snapshot
response = ffi.new('msg_appendentries_response_t*')
response.success = 1
response.current_idx = snapshot.last_idx
response.term = lib.raft_get_current_term(self.raft)
response.first_idx = response.current_idx
self.network.enqueue_msg(response, self, other)
node_id = lib.raft_get_nodeid(self.raft)
# set membership configuration according to snapshot
for member in snapshot.members:
if -1 == member.id:
continue
node = lib.raft_get_node(self.raft, member.id)
if not node:
udata = ffi.NULL
try:
node_sv = self.network.id2server(member.id)
udata = node_sv.udata
except ServerDoesNotExist:
pass
node = lib.raft_add_node(self.raft, udata, member.id, member.id == node_id)
lib.raft_node_set_active(node, 1)
if member.voting and not lib.raft_node_is_voting(node):
lib.raft_node_set_voting(node, 1)
elif not member.voting and lib.raft_node_is_voting(node):
lib.raft_node_set_voting(node, 0)
if node_id != member.id:
assert node
# TODO: this is quite ugly
# we should have a function that removes all nodes by ourself
# if (!raft_get_my_node(self->raft)) */
# raft_add_non_voting_node(self->raft, NULL, node_id, 1); */
e = lib.raft_end_load_snapshot(self.raft)
assert(0 == e)
assert(lib.raft_get_log_count(self.raft) == 0)
self.do_membership_snapshot()
self.snapshot.image = dict(snapshot.image)
self.snapshot.last_term = snapshot.last_term
self.snapshot.last_idx = snapshot.last_idx
assert(lib.raft_get_my_node(self.raft))
# assert(sv->snapshot_fsm);
self.fsm_dict = dict(snapshot.image)
logging.warning('{} loaded snapshot t:{} idx:{}'.format(
self, snapshot.last_term, snapshot.last_idx))
def recv_entry(self, ety):
# FIXME: leak
response = ffi.new('msg_entry_response_t*')
return lib.raft_recv_entry(self.raft, ety, response)
def toggle_membership(self):
if not self.leader:
logging.error('no leader')
return
leader = self.leader
server = self.random.choice(self.active_servers)
if not lib.raft_is_leader(leader.raft):
return
if lib.raft_voting_change_is_in_progress(leader.raft):
# logging.error('{} voting change in progress'.format(server))
return
if leader == server:
# logging.error('can not remove leader')
return
if server.connection_status in [NODE_CONNECTING, NODE_DISCONNECTING]:
# logging.error('can not remove server that is changing connection status')
return
if NODE_DISCONNECTED == server.connection_status:
self.remove_server(server)
server = RaftServer(self)
# Create a new configuration entry to be processed by the leader
ety = ffi.new('msg_entry_t*')
ety.term = 0
ety.id = self.new_entry_id()
if server.connection_status == NODE_CONNECTED:
ety.type = lib.RAFT_LOGTYPE_DEMOTE_NODE
else:
ety.type = lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE
change = ffi.new_handle(ChangeRaftEntry(server.id))
ety.data.buf = change
ety.data.len = ffi.sizeof(ety.data.buf)
assert(lib.raft_entry_is_cfg_change(ety))
self.entries.append((ety, change))
# logging.warning("{}> {} Membership change {}: ety.id:{} id:{} {}".format(
# self.iteration,
# server,
# "ADD" if ety.type == lib.RAFT_LOGTYPE_ADD_NONVOTING_NODE else "DEMOTE",
# ety.id,
# lib.raft_get_nodeid(leader.raft),
# server.id,
# ))
e = leader.recv_entry(ety)
if 0 != e:
logging.error(err2str(e))
return
else:
self.num_membership_changes += 1
# Wake up new node
if NODE_DISCONNECTED == server.connection_status:
server.set_connection_status(NODE_CONNECTING)
assert server.udata
added_node = lib.raft_add_non_voting_node(server.raft, server.udata, server.id, 1)
assert added_node
elif NODE_CONNECTED == server.connection_status:
server.set_connection_status(NODE_DISCONNECTING)
else:
logging.warning('Unknown status set {}'.format(server.connection_status))