def test_candidate_server_wins_election(self):
board = MemoryBoard()
state = Follower()
server0 = Server(0, state, [], board, [])
board = MemoryBoard()
state = Follower()
oserver = Server(1, state, [], board, [])
board = MemoryBoard()
state = Candidate()
server = Server(2, state, [], board, [oserver, server0])
server0._neighbors.append(server)
oserver._neighbors.append(server)
oserver.on_message(oserver._messageBoard.get_message())
server0.on_message(server0._messageBoard.get_message())
server._total_nodes = 3
server.on_message(server._messageBoard.get_message())
server.on_message(server._messageBoard.get_message())
self.assertEqual(type(server._state), Leader)
def test_two_candidates_tie(self):
followers = []
for i in range(4):
board = MemoryBoard()
state = Follower()
followers.append(Server(i, state, [], board, []))
board = MemoryBoard()
state = Candidate()
c0 = Server(5, state, [], board, followers[0:2])
board = MemoryBoard()
state = Candidate()
c1 = Server(6, state, [], board, followers[2:])
for i in range(2):
followers[i]._neighbors.append(c0)
followers[i].on_message(followers[i]._messageBoard.get_message())
for i in range(2, 4):
followers[i]._neighbors.append(c1)
followers[i].on_message(followers[i]._messageBoard.get_message())
c0._total_nodes = 6
c1._total_nodes = 6
for i in range(2):
c0.on_message(c0._messageBoard.get_message())
c1.on_message(c1._messageBoard.get_message())
self.assertEqual(type(c0._state), Candidate)
self.assertEqual(type(c1._state), Candidate)
def setUp(self):
board = MemoryBoard()
state = Follower()
self.oserver = Server(0, state, [], board, [])
board = MemoryBoard()
state = Follower()
self.server = Server(1, state, [], board, [self.oserver])
def __init__(self, config, server_id):
"""
Initialize the class using the config file provided and also initialize
any data structures you may need.
:param config:
"""
with open(config, 'r') as f:
lines = f.readlines()
self.n_nodes = int(lines[0].split()[1])
self.nodes = []
self.nodes_conns = [None] * self.n_nodes # RPC Connections
for _id in range(self.n_nodes):
_host, _port = lines[_id + 1].split(' ')[1].split(':')
_port = int(_port)
self.nodes.append((_id, _host, _port))
self.lock = Lock()
self.name = server_id
# TODO: File
use_persistent = False
if use_persistent:
current_term_filename = '/tmp/%d-current_term.txt' % self.name
if os.path.isfile(current_term_filename):
with open(current_term_filename, 'r') as f:
self.current_term = eval(f.read())
else:
self.current_term = 0
with open(current_term_filename, 'w') as f:
f.write(str(self.current_term))
voted_for_filename = '/tmp/%d-voted_for.txt' % self.name
if os.path.isfile(voted_for_filename):
with open(voted_for_filename, 'r') as f:
self.voted_for = eval(f.read())
else:
# This is used to ensure a voter only votes for one node in a term
self.voted_for = None
with open('/tmp/%d-voted_for.txt' % self.name, 'w') as f:
f.write(str(self.voted_for))
else:
self.voted_for = None
self.current_term = 0
self.state = Follower(self)
self.state.set_server(self)
# A queue of event messages
self.message_inbox = MemoryBoard()
self.message_inbox.set_owner(self)
self.consume_inbox_thread = Thread(target=self.consume_inbox)
self.consume_inbox_thread.daemon = True
self.consume_inbox_thread.start()
def setUp(self):
board = MemoryBoard()
state = Follower()
self.oserver = Server(0, state, [], board, [])
board = MemoryBoard()
state = Candidate()
self.server = Server(1, state, [], board, [self.oserver])
self.oserver._neighbors.append(self.server)
def setUp(self):
followers = []
for i in range(1, 4):
board = MemoryBoard()
state = Follower()
followers.append(Server(i, state, [], board, []))
board = MemoryBoard()
state = Leader()
self.leader = Server(0, state, [], board, followers)
for i in followers:
i._neighbors.append(self.leader)
def setUpClass(self):
self.servers = []
for i in range(N):
if i == 0:
state = Leader()
else:
state = Follower()
s = ZeroMQServer("S%d" % i, state, [], MemoryBoard(), [], 6666 + i)
if (i == 0):
self.leader = s
self.servers.append(s)
for i in range(N):
me = self.servers[i]
neighbors = [n for n in self.servers if n != me]
me.set_neighbors(neighbors)
class TestMemoryBoard(unittest.TestCase):
def setUp(self):
self.board = MemoryBoard()
def test_memoryboard_post_message(self):
msg = BaseMessage(0, 0, 0, 0)
self.board.post_message(msg)
self.assertEqual(msg, self.board.get_message())
def test_memoryboard_post_message_make_sure_they_are_ordered(self):
msg = BaseMessage(0, 0, 0, 0)
msg2 = BaseMessage(0, 0, 0, 0)
msg2._timestamp -= 100
self.board.post_message(msg)
self.board.post_message(msg2)
self.assertEqual(msg2, self.board.get_message())
class TestMemoryBoard(unittest.TestCase):
def setUp(self):
self.board = MemoryBoard()
def test_memoryboard_post_message(self):
msg = BaseMessage(0, 0, 0, 0)
self.board.post_message(msg)
self.assertEqual(msg, self.board.get_message())
def test_memoryboard_post_message_make_sure_they_are_ordered(self):
msg = BaseMessage(0, 0, 0, 0)
msg2 = BaseMessage(0, 0, 0, 0)
msg2._timestamp -= 100
self.board.post_message(msg)
self.board.post_message(msg2)
self.assertEqual(msg2, self.board.get_message())
# Code is based on this website
# https://www.bogotobogo.com/python/python_network_programming_client_client.php
# run_client.py
# This file mainly deal listening to network and keyboard, and take care of
# timeout event.
import socket
import datetime
import time
import sys
import threading
from client_config import CONFIG as client_config
from clients.client import Client, ZeroMQclient
#from states.follower import Follower
from boards.memory_board import MemoryBoard
from states.characters import Follower
if __name__ == '__main__':
client_id = sys.argv[1]
ZeroMQclient('localhost\t' + str(client_config[client_id]), client_id,
MemoryBoard())
while True:
time.sleep(10)
import socket
import datetime
import time
import sys
import threading
from server_config import CONFIG as server_config
from servers.server import Server, ZeroMQServer
#from states.follower import Follower
from blocks.block import BlockChain
from boards.memory_board import MemoryBoard
import logging
from states.characters import Follower
if __name__ == '__main__':
server_id = int(sys.argv[1])
logging.basicConfig(filename='server_' + str(server_id) + '.log',
level=logging.DEBUG)
observers = [
Server('localhost\t' + str(server_config[i]), Follower(), BlockChain(),
MemoryBoard(), [], True) for i in server_config
if i != server_id
]
#for this_item in observers:
# this_item._state.set_server(this_item)
ZeroMQServer('localhost\t' + str(server_config[server_id]), Follower(),
BlockChain(), MemoryBoard(), observers)
#ZeroMQServer(this_server)
while True:
time.sleep(10)
import unittest from boards.memory_board import MemoryBoard from messages.append_entries import AppendEntriesMessage from messages.request_vote import RequestVoteMessage from servers.server import Server from states.follower import Follower from states.candidate import Candidate from states.leader import Leader board = MemoryBoard() state = Follower() oserver = Server(0, state, [], board, []) # print(oserver) """ _name: 0, _state: Follower ** _timeout: 500, _timeoutTime: 1581685132.670627, _last_vote: None, _log: [], _messageBoard: board: [], _neighbors: [], _total_nodes: 0, _commitIndex: 0, _currentTerm: 0, _lastApplied: 0, _lastLogIndex: 0, _lastLogTerm: None """ board = MemoryBoard() state = Candidate() # 候选人 # print(state) # Candidate ** _last_vote: None, _votes: None
class RaftNode(rpyc.Service):
"""
A RAFT RPC server class.
"""
def __init__(self, config, server_id):
"""
Initialize the class using the config file provided and also initialize
any data structures you may need.
:param config:
"""
with open(config, 'r') as f:
lines = f.readlines()
self.n_nodes = int(lines[0].split()[1])
self.nodes = []
self.nodes_conns = [None] * self.n_nodes # RPC Connections
for _id in range(self.n_nodes):
_host, _port = lines[_id + 1].split(' ')[1].split(':')
_port = int(_port)
self.nodes.append((_id, _host, _port))
self.lock = Lock()
self.name = server_id
# TODO: File
use_persistent = False
if use_persistent:
current_term_filename = '/tmp/%d-current_term.txt' % self.name
if os.path.isfile(current_term_filename):
with open(current_term_filename, 'r') as f:
self.current_term = eval(f.read())
else:
self.current_term = 0
with open(current_term_filename, 'w') as f:
f.write(str(self.current_term))
voted_for_filename = '/tmp/%d-voted_for.txt' % self.name
if os.path.isfile(voted_for_filename):
with open(voted_for_filename, 'r') as f:
self.voted_for = eval(f.read())
else:
# This is used to ensure a voter only votes for one node in a term
self.voted_for = None
with open('/tmp/%d-voted_for.txt' % self.name, 'w') as f:
f.write(str(self.voted_for))
else:
self.voted_for = None
self.current_term = 0
self.state = Follower(self)
self.state.set_server(self)
# A queue of event messages
self.message_inbox = MemoryBoard()
self.message_inbox.set_owner(self)
self.consume_inbox_thread = Thread(target=self.consume_inbox)
self.consume_inbox_thread.daemon = True
self.consume_inbox_thread.start()
def send_message_to_all(self, message, receivers=None):
if receivers is None:
receivers = [node[0] for node in self.nodes]
def send_one_message(_message, _node_name, _host, _port):
_message.receiver = _node_name # The id of the receiver
try:
if self.nodes_conns[_message.receiver] is None:
self.nodes_conns[_message.receiver] = rpyc.connect(
_host, _port, config={'allow_pickle': True})
# async_post_message = rpyc.async_(conn.root.post_message)
# async_post_message(_message)
self.nodes_conns[_message.receiver].root.post_message(_message)
except ConnectionRefusedError:
self.nodes_conns[_message.receiver] = None
except EOFError:
self.nodes_conns[_message.receiver] = None
# logging.warning('Connection to server #%d refused.' % _node_name)
for node_name in receivers:
_, _host, _port = self.nodes[node_name]
Thread(target=send_one_message,
args=(copy.deepcopy(message), node_name, _host,
_port)).start()
# Thread(target=send_one_message, args=(message, node_name, _host, _port)).start()
# try:
# conn = rpyc.connect(_host, _port, config={'allow_pickle': True})
# message.receiver = node_name # The id of the receiver
# async_post_message = rpyc.async_(conn.root.post_message)
# async_post_message(message)
# # conn.root.post_message(message)
# # conn.close()
# except ConnectionRefusedError:
# logging.debug('Connection to server #%d refused.' % node_name)
def send_message_response(self, message):
# logging.warning('Server #%d sending Message: %s' % (self.name, message))
def send_one_message(_message, _node_name, _host, _port):
try:
if self.nodes_conns[_message.receiver] is None:
self.nodes_conns[_message.receiver] = rpyc.connect(
_host, _port, config={'allow_pickle': True})
_message.receiver = _node_name # The id of the receiver
# async_post_message = rpyc.async_(conn.root.post_message)
# async_post_message(_message)
self.nodes_conns[message.receiver].root.post_message(_message)
logging.warning('Responded to #%d (host: %s, port: %d)' %
(_message.receiver, _host, _port))
except ConnectionRefusedError:
self.nodes_conns[message.receiver] = None
except EOFError:
self.nodes_conns[message.receiver] = None
# logging.warning('Connection to server #%d refused. [In send_message_response]' % _node_name)
_node_name, _host, _port = self.nodes[message.receiver]
Thread(target=send_one_message,
args=(message, _node_name, _host, _port)).start()
# try:
# _, _host, _port = self.nodes[message.receiver]
# conn = rpyc.connect(_host, _port, config={'allow_pickle': True})
# async_post_message = rpyc.async_(conn.root.post_message)
# async_post_message(message)
# # conn.root.post_message(message)
# # conn.close()
# except ConnectionRefusedError:
# logging.debug('Connection to server #%d refused.' % message.receiver)
def consume_inbox(self):
from queue import Empty
while True:
try:
# get_message() operation is blocking
# i.e. block if inbox is empty until an item is available
message = self.message_inbox.get_message()
logging.warning('Consuming: %s' % message)
self.state.on_message(message)
except Empty:
pass
def exposed_post_message(self, message):
"""
Allows clients (other servers) to sent message to this server.
:param message:
:return:
"""
message = copy.deepcopy(message)
with self.lock:
self.message_inbox.post_message(message)
def exposed_is_leader(self):
"""
x = is_leader(): returns True or False, depending on whether
this node is a leader
As per rpyc syntax, adding the prefix 'exposed_' will expose this
method as an RPC call
:return:
"""
with self.lock:
return isinstance(self.state, Leader)
from boards.memory_board import MemoryBoard
from messages.base import BaseMessage
board = MemoryBoard()
print(board)
msg = BaseMessage(0, 0, 0, 0)
print(msg)
board.post_message(msg)
for msg in board._board:
print(msg)
print(board.get_message())
"""
board: []
AppendEntries: 0, RequestVote: 1, RequestVoteResponse: 2, Response: 3, _timestamp: 1581641629, _sender: 0, _receiver: 0, _data: 0, _term: 0
AppendEntries: 0, RequestVote: 1, RequestVoteResponse: 2, Response: 3, _timestamp: 1581641629, _sender: 0, _receiver: 0, _data: 0, _term: 0
AppendEntries: 0, RequestVote: 1, RequestVoteResponse: 2, Response: 3, _timestamp: 1581641629, _sender: 0, _receiver: 0, _data: 0, _term: 0
huzhi@huzhideMacBook-Pro simpleRaft %
"""
def setUp(self):
self.board = MemoryBoard()
def setUp(self):
self.board = MemoryBoard()
