def reBootstrap(self, bootstrapNodeLocList, enclaveId):
'''Try to re-bootstrap into the network being managed by bootstrapNodeLoc'''
try:
glog.debug("SuccessorsList: all successors are dead... attempting to re-bootstrap", system=str(self.nodeLocation))
# If I am my own successor but I'm not a bootstrap node, try to bootstrap
# back into the network.
# Try to get a connection to any of the bootstrap nodes.
# if bootstrapNodeLoc is not None and \
# bootstrapNodeLoc.id != self.nodeLocation.id:
# We have no successors, so we'd better bootstrap back in.
(bootstrapNode, factory, conn, nodeLocation) = yield Utils.getRemoteConnectionFromList(bootstrapNodeLocList, metricsMessageCounter=self.metricsMessageCounter)
if bootstrapNode is False:
# Could not get a connection
# DPF - This is a change on Oct, 2014 -- even if I am NOT a bootstrapNode,
# - I will add myself as a successor. Need to see if this breaks stuff.
# - old version only did this for bootstrap nodes.
self.addValues(self.nodeLocation)
else:
succLoc = yield bootstrapNode.callRemote("findSuccessorLocation", self.nodeLocation.id, enclaveId)
if succLoc is not None:
(factory, conn2) = Utils.getRemoteConnection(succLoc, self.metricsMessageCounter)
succNode = yield factory.getRootObject()
successorsSuccessorList = yield succNode.callRemote("getSuccessorList", enclaveId)
# Add the successor to my list
self.addValues(succLoc)
# Add the successorsList to my list also (for efficiency)
self.addValues(successorsSuccessorList)
# Close connections
Utils.disconnect(None, conn2)
# Close the connections
Utils.disconnect(None, conn)
except Exception, e:
log.msg("bootstrap node is dead. Try again later. [%s]" % e, system="SuccessorsList")
log.err(e, "bootstrap node is dead. Try again later.", system="SuccessorsList")
def rebuildSuccessorList(self, enclaveId, bootstrapNodeLocList = None):
# Track changes to successor for debugging
origSuccessor = self.getSuccessor() # Solely for debugging when successors change
# Prune my successor's list for "alive" only
yield self.removeDeadSuccessors()
subFromI = 0
for i in range(len(self.theList)):
currentSuccLocation = self.theList[i-subFromI]
try:
# Check if the succesor's pred is currentPred
(factory, conn) = Utils.getRemoteConnection(currentSuccLocation, self.metricsMessageCounter)
succ = yield factory.getRootObject()
successorsSuccessorList = yield succ.callRemote("getSuccessorList", enclaveId)
# Add them to my successorsList
self.addValues(successorsSuccessorList)
# Close the connection
Utils.disconnect(None, conn)
# Once we find one live successor list, it's good enough. We can go.
#glog.debug("rebuildSuccessorList: connected successor is: %s" % currentSuccLocation)
defer.returnValue(True)
except Exception :
# Successor wasn't alive, remove from my list of successors
glog.debug("Removing dead successor during rebuild location: ID %s" % currentSuccLocation.id, system='rebuildSuccessorList')
subFromI = self.removeLocationFromList(currentSuccLocation)
# Attempt to re-bootstrap in
yield self.reBootstrap(bootstrapNodeLocList, enclaveId)
self.checkFinalSuccessor(origSuccessor, "rebuildSuccessorList") #DEBUG
defer.returnValue(True)
