class Server(BaseActor):
def __init__(self, servers, sId, stateFileName):
BaseActor.__init__(self, servers, sId)
self.stateFileName = stateFileName
self.SERVER_IP = None
self.UDP_PORT = None
self.TCP_PORT = None
self.udpSock = None
self.tcpSock = None
self.inputs = None
self.currentRequests = set()
self.requestQueue = queue.Queue()
self.election = Election(servers, sId)
self.recovery = True
self.recoveryLookahead = True
self.inRecovery = set()
self.lookaheadIndex = None
self.recoveryTimer = None
self.proposer = None
self.acceptor = None
self.learner = None
# writes current state of node to disk
def _saveState(self):
acceptorDump = self.acceptor.exportDict()
learnerDump = self.learner.exportDict()
dump = dict()
dump['acceptor'] = acceptorDump
dump['learner'] = learnerDump
with open(self.stateFileName, 'wb') as sf:
pickle.dump(dump, sf)
# load state from disk
def _loadState(self):
with open(self.stateFileName, 'rb') as sf:
dump = pickle.load(sf)
acceptorDump = dump['acceptor']
learnerDump = dump['learner']
self.acceptor.importDict(acceptorDump)
self.learner.importDict(learnerDump)
def _initActors(self):
self.proposer = Proposer(self.servers, self.id)
self.acceptor = Acceptor(self.servers, self.id)
self.learner = Learner(self.servers, self.id)
# load saved
if os.path.isfile(self.stateFileName):
print('Loading from:', self.stateFileName)
self._loadState()
# set up networking
def _initNetworking(self):
self.SERVER_IP, self.UDP_PORT, self.TCP_PORT = self.servers[self.id]
print(self.SERVER_IP, self.UDP_PORT, self.TCP_PORT, self.id)
self.udpSock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.udpSock.bind((self.SERVER_IP, self.UDP_PORT))
self.tcpSock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.tcpSock.bind((self.SERVER_IP, self.TCP_PORT))
self.tcpSock.listen(5)
self.inputs = [ self.udpSock, self.tcpSock ]
# initialize actors and networking
def init(self):
self._initActors()
self._initNetworking()
# if not leader, forward message to the leader
# if no leader, initiate leader election
def forwardToLeader(self, msg):
leader = self.election.getLeader()
if leader is None:
self.election.startElection()
else:
send(self.servers[leader], msg)
def handleRequest(self, msg):
# is the leader, check is already complete, add to queue, propose when ready
if self.election.isLeader():
self.requestQueue.put(msg)
else:
self.forwardToLeader(msg)
def handleLog(self, msg):
if self.election.isLeader():
log = sorted([ (k, v) for k, v in self.learner.learnedValues.items()])
for entry in log:
print(entry)
else:
self.forwardToLeader(msg)
# find the current head of the log by creating proposals for the next slot until head is found
def doLookahead(self):
if not self.recoveryLookahead:
self.lookaheadIndex = None
return
if not self.election.isLeader():
return
currentMaxIndex = self.learner.getMaxIndex()
if currentMaxIndex is None:
if self.lookaheadIndex is None:
self.lookaheadIndex = self.learner.getBaseIndex()
msg = self.createEmptyRequest(self.lookaheadIndex)
self.proposer.newReqProposal(msg, self.lookaheadIndex)
elif self.lookaheadIndex is None or self.lookaheadIndex <= currentMaxIndex:
self.lookaheadIndex = currentMaxIndex + 1
msg = self.createEmptyRequest(self.lookaheadIndex)
self.proposer.newReqProposal(msg, self.lookaheadIndex)
elif self.lookaheadIndex == currentMaxIndex + 1:
# still at current lookahead
return
else:
print('Error: Lookahead is too far ahead', file=sys.stderr)
# recover known missing slots in the log
def doRecovery(self):
self.startRecoveryTimer()
if not self.recovery and not self.recoveryLookahead:
return
# if not leader, stop with recovery
if not self.election.isLeader():
return
missing = self.learner.getMissingValues()
print('Missing:', missing)
print('In Recovery:', self.inRecovery)
print('Perform Lookahead:', self.recoveryLookahead)
print('Lookahead Index:', self.lookaheadIndex)
if len(missing) == 0:
self.recovery = False
self.inRecovery.clear()
# we're done with recovery, do lookahead if not done
self.doLookahead()
return
# if we need recovery, do recovery also
self.recoveryLookahead = True
for index in missing:
# already in recovery
if index in self.inRecovery:
continue
self.inRecovery.add(index)
# create dummy message to fill in gap
msg = self.createEmptyRequest(index)
self.proposer.newReqProposal(msg, index)
def resetRecoveryState(self):
self.recovery = True
def startRecoveryTimer(self):
if self.recoveryTimer is not None and self.recoveryTimer.is_alive():
return
self.recoveryTimer = threading.Timer(5.0, self.resetRecoveryState)
self.recoveryTimer.start()
def createEmptyRequest(self, index):
msg = dict()
msg['type'] = 'request'
msg['clientid'] = self.id*-1 # negative server id will indicate server request
msg['reqid'] = index # fake reqid
msg['value'] = ""
return msg
def processRequest(self):
if self.recovery or self.recoveryLookahead or self.requestQueue.empty():
return
msg = self.requestQueue.get()
clientId = msg['clientid']
clientReqId = msg['reqid']
gReqId = getGlobalReqId(clientId, clientReqId)
completed = self.learner.getCompleted(gReqId)
if completed is not None:
clientRetIP = msg.get('retip') #str
clientRetPort = msg.get('retport') #int
self.sendReply(clientRetIP, clientRetPort, completed)
self.processRequest()
return
index = self.learner.getNextIndex()
self.proposer.newReqProposal(msg, index)
def handleAccepted(self, msg):
resultLearner = self.learner.handleAccepted(msg)
acceptedProposal = self.proposer.handleAccepted(msg)
if resultLearner is False or acceptedProposal is None:
return
if resultLearner != acceptedProposal.isLearned():
print('Error: Learner and Proposer disagree', acceptedProposal.isLearned(), file=sys.stderr)
wasOverridden = acceptedProposal.isOverridden()
origRetried = acceptedProposal.isOrigRetried()
if wasOverridden and not origRetried:
retryMsg = acceptedProposal.recreateOrigRequest()
# if request was dummy, don't resubmit
if retryMsg.get('clientid') < 0:
acceptedProposal.setOrigRetried()
return
self.requestQueue.put(retryMsg)
acceptedProposal.setOrigRetried()
elif not wasOverridden:
self.recoveryLookahead = False
retIp, retPort = acceptedProposal.getReturnInfo()
index = acceptedProposal.getIndex()
if not self.learner.checkReply(index):
self.sendReply(retIp, retPort, index)
def sendReply(self, retIp, retPort, index):
if retIp is None or retPort is None:
return
print('Sent Reply', retIp, retPort)
value = self.learner.getLearnedValue(index)
omsg = dict()
omsg['type'] = 'response'
omsg['value'] = value
send((retIp, retPort, None), omsg)
self.learner.addReply(index)
def handleMsg(self, msg):
if msg is None:
return
msgType = msg['type']
if msgType == 'request':
print('request')
self.handleRequest(msg)
elif msgType == 'log':
print('log')
self.handleLog(msg)
elif msgType == 'prepare':
print('prepare')
print(msg)
self.acceptor.handlePrepare(msg)
elif msgType == 'promise':
print('promise')
result = self.proposer.handlePromise(msg)
if result:
self.recoveryLookahead = False
elif msgType == 'accept':
print('accept')
self.acceptor.handleAccept(msg)
elif msgType == 'accepted':
print('accepted')
self.handleAccepted(msg)
else:
print('msgType not recognized', file=sys.stderr)
def handleElectionMsg(self, msg):
if msg is None:
return
msgType = msg['type']
if msgType == 'election':
self.election.handleElection(msg)
elif msgType == 'answer':
self.election.handleAnswer(msg)
elif msgType == 'coordinator':
self.election.handleCoordinator(msg)
self.proposer.setLeader(self.election.isLeader())
print('Leader is:', self.election.getLeader())
else:
print('msgType not recognized', file=sys.stderr)
def run(self):
if self.election.getLeader() is None:
self.election.startElection()
while True:
readable, writable, exceptional = select.select(self.inputs, [], [], 1)
for s in readable:
# tcpSock
if s is self.tcpSock:
conn, addr = s.accept()
msg = recv(conn)
print('Received', msg)
conn.close()
self.handleElectionMsg(msg)
# udpSock
else:
msg = recv(s)
self.handleMsg(msg)
self._saveState()
self.doRecovery()
self.processRequest()
