def reBootstrap(self, bootstrapNodeLocList, enclaveId): '''Try to re-bootstrap into the network being managed by bootstrapNodeLoc''' try: glog.debug("SuccessorsList: all successors are dead... attempting to re-bootstrap", system=str(self.nodeLocation)) # If I am my own successor but I'm not a bootstrap node, try to bootstrap # back into the network. # Try to get a connection to any of the bootstrap nodes. # if bootstrapNodeLoc is not None and \ # bootstrapNodeLoc.id != self.nodeLocation.id: # We have no successors, so we'd better bootstrap back in. (bootstrapNode, factory, conn, nodeLocation) = yield Utils.getRemoteConnectionFromList(bootstrapNodeLocList, metricsMessageCounter=self.metricsMessageCounter) if bootstrapNode is False: # Could not get a connection # DPF - This is a change on Oct, 2014 -- even if I am NOT a bootstrapNode, # - I will add myself as a successor. Need to see if this breaks stuff. # - old version only did this for bootstrap nodes. self.addValues(self.nodeLocation) else: succLoc = yield bootstrapNode.callRemote("findSuccessorLocation", self.nodeLocation.id, enclaveId) if succLoc is not None: (factory, conn2) = Utils.getRemoteConnection(succLoc, self.metricsMessageCounter) succNode = yield factory.getRootObject() successorsSuccessorList = yield succNode.callRemote("getSuccessorList", enclaveId) # Add the successor to my list self.addValues(succLoc) # Add the successorsList to my list also (for efficiency) self.addValues(successorsSuccessorList) # Close connections Utils.disconnect(None, conn2) # Close the connections Utils.disconnect(None, conn) except Exception, e: log.msg("bootstrap node is dead. Try again later. [%s]" % e, system="SuccessorsList") log.err(e, "bootstrap node is dead. Try again later.", system="SuccessorsList")
def rebuildSuccessorList(self, enclaveId, bootstrapNodeLocList = None): # Track changes to successor for debugging origSuccessor = self.getSuccessor() # Solely for debugging when successors change # Prune my successor's list for "alive" only yield self.removeDeadSuccessors() subFromI = 0 for i in range(len(self.theList)): currentSuccLocation = self.theList[i-subFromI] try: # Check if the succesor's pred is currentPred (factory, conn) = Utils.getRemoteConnection(currentSuccLocation, self.metricsMessageCounter) succ = yield factory.getRootObject() successorsSuccessorList = yield succ.callRemote("getSuccessorList", enclaveId) # Add them to my successorsList self.addValues(successorsSuccessorList) # Close the connection Utils.disconnect(None, conn) # Once we find one live successor list, it's good enough. We can go. #glog.debug("rebuildSuccessorList: connected successor is: %s" % currentSuccLocation) defer.returnValue(True) except Exception : # Successor wasn't alive, remove from my list of successors glog.debug("Removing dead successor during rebuild location: ID %s" % currentSuccLocation.id, system='rebuildSuccessorList') subFromI = self.removeLocationFromList(currentSuccLocation) # Attempt to re-bootstrap in yield self.reBootstrap(bootstrapNodeLocList, enclaveId) self.checkFinalSuccessor(origSuccessor, "rebuildSuccessorList") #DEBUG defer.returnValue(True)