class Server(object): def __init__(self, peers, host, port): self.host = host self.port = port self.peer_id = '{}:{}'.format(host, port) self._logger = logging.getLogger(__name__) self._loop = asyncio.get_event_loop() self._pool = Pool(self, peers) # heartbeat constants and bookkeeping variables self._heartbeat_interval = 1000 # ms self._last_interval = None self._min_heartbeat_timeout = 2000 # ms self._max_heartbeat_timeout = 4000 # ms self._heartbeat_timeout = None self._last_heartbeat = None self.reset_heartbeat() self.reset_timeout() self._log = Log(Machine()) self.state = State.FOLLOWER self.term = 0 self.voted = None self.votes = set() self._pending_clients = {} self.handlers = {'append_entries_req': self.handle_append_entries_req, 'append_entries_resp': self.handle_append_entries_resp, 'request_vote_req': self.handle_request_vote_req, 'request_vote_resp': self.handle_request_vote_resp} def reset_heartbeat(self): self._last_heartbeat = self._loop.time() def reset_interval(self): self._last_interval = self._loop.time() def reset_timeout(self): self._heartbeat_timeout = randint(self._min_heartbeat_timeout, self._max_heartbeat_timeout) / 1000 @property def stale(self): return self._last_heartbeat + self._heartbeat_timeout < self._loop.time() @staticmethod def decode(data): return json.loads(data.decode()) @staticmethod def encode(data): return json.dumps(data).encode() def broadcast(self, request): for peer in self._pool: self.send_async(peer, request) @asyncio.coroutine def run(self): self.reset_interval() while True: self._logger.debug('state: {}, term: {}'.format(self.state, self.term)) if self.state == State.LEADER: self.append_entries() if self.state in (State.CANDIDATE, State.FOLLOWER) and self.stale: self.request_vote() yield from self.wait() @asyncio.coroutine def send(self, peer, request): """ Send a request to a peer (if available). """ transport = yield from peer.get_transport() if transport: transport.write(self.encode(request)) def send_async(self, peer, request): """ Schedule the execution """ asyncio.async(self.send(peer, request)) @asyncio.coroutine def wait(self): """ Wait for the next interval. """ tic = self._heartbeat_interval / 1000 - self._loop.time() + self._last_interval yield from asyncio.sleep(tic) self.reset_interval() def to_leader(self): self.append_entries() self.state = State.LEADER self.voted = None self.votes = set() for peer in self._pool: peer.match = -1 peer.next = self._log.index + 1 def to_follower(self, term): self.state = State.FOLLOWER self.term = term self.voted = None self.votes = set() def append_entries(self, peer=None): """ Append entries RPC. """ peers = self._pool.all() if peer is None else [peer] for peer in peers: log_entries = self._log[peer.next:] log_index, log_term, _ = self._log[peer.next - 1] request = {'rpc': 'append_entries_req', 'peer_id': self.peer_id, 'term': self.term, 'log_commit': self._log.commit, 'log_entries': log_entries, 'log_index': log_index, 'log_term': log_term, } self.send_async(peer, request) self._logger.debug('broadcasting append entries') def request_vote(self): """ Request vote RPC. """ self.reset_heartbeat() self.reset_timeout() self.state = State.CANDIDATE self.term += 1 self.voted = self.peer_id self.votes = set([self.peer_id]) request = {'rpc': 'request_vote_req', 'peer_id': self.peer_id, 'term': self.term, 'log_index': self._log.index, 'log_term': self._log.term, } self.broadcast(request) self._logger.debug('broadcasting request vote') def handle_peer(self, request): """ Dispatch requests to the appropriate handlers. """ if self.term < request['term']: self.to_follower(request['term']) return self.handlers[request['rpc']](request) def handle_append_entries_req(self, request): self._logger.debug('append entries request received') if request['term'] < self.term: return self.reset_heartbeat() log_index = request['log_index'] log_term = request['log_term'] if not self._log.match(log_index, log_term): return {'rpc': 'append_entries_resp', 'peer_id': self.peer_id, 'term': self.term, 'log_index': self._log.index, 'success': False } log_entries = request['log_entries'] self._log.append(log_index, log_entries) log_commit = request['log_commit'] if self._log.commit < log_commit: index = min(self._log.index, log_commit) self._log.commit = index self._log.apply(index) if not log_entries: # no need to answer, the peer might have committed return # new entries but has certainly not replicated new ones return {'rpc': 'append_entries_resp', 'peer_id': self.peer_id, 'term': self.term, 'log_index': self._log.index, 'log_term': self._log.term, 'success': True, } def handle_append_entries_resp(self, response): if response['success']: self._logger.debug('append entries succeeded') log_index = response['log_index'] log_term = response['log_term'] peer_id = response['peer_id'] self._pool[peer_id].match = log_index self._pool[peer_id].next = log_index + 1 if (self._log.commit < log_index and self._pool.ack(log_index) and log_term == self.term): self._log.commit = log_index results = self._log.apply(log_index) self.return_results(results) else: peer = self._pool[response['peer_id']] peer.next -= 1 self.append_entries(peer) # self._logger.debug('append entries failed') def handle_request_vote_req(self, request): self._logger.debug('request vote request received') if request['term'] < self.term: return log_index = request['log_index'] log_term = request['log_term'] peer_id = request['peer_id'] if self.voted in (None, peer_id) and self._log.match(log_index, log_term): granted = True self.reset_heartbeat() else: granted = False return {'rpc': 'request_vote_resp', 'peer_id': self.peer_id, 'term': self.term, 'granted': granted, } def handle_request_vote_resp(self, response): if self.term == response['term'] and response['granted']: self.votes.add(response['peer_id']) if self._pool.majority(len(self.votes)): self.to_leader() def handle_client(self, cmd, transport): self._log.add(self.term, cmd) self._pending_clients[(self.term, self._log.index)] = transport self.append_entries() def return_results(self, results): for result in results: term, index, result = result transport = self._pending_clients.pop((term, index)) transport.write(self.encode(result)) transport.close()