def onResponsibilityChange(self):
#print("BACKUP IS HAPPENING")
records = list(filter(self.doIOwn, self.database.getRecords()))
##print(records)
rlocs = list(map(lambda x: space.idToPoint(2, x), records))
##print(rlocs)
lMap = {}
for l,p in zip(rlocs,records):
lMap[l]=p
candidates = None
with self.peersLock:
candidates = self.seekCandidates[:]
if candidates is None:
return
candidates.remove(self.loc)
#print("canidates",candidates)
if len(candidates) == 0:
return #im alone, no backups to send
for loc in rlocs:
l = lMap[loc]
bestLoc = space.getClosest(loc, candidates)
peer = self.locPeerDict[bestLoc]
value = self.database.get(l)
#print(loc,l,peer)
try:
self.network.store(self.key,peer,l,value)#this backs up existing values, and stores old values on new nodes.
#print("%s is backed up to %s"%(l,peer))
except Exception as e:
print(e)
continue
def onResponsibilityChange(self):
#print("BACKUP IS HAPPENING")
records = list(filter(self.doIOwn, self.database.getRecords()))
##print(records)
rlocs = list(map(lambda x: space.idToPoint(2, x), records))
##print(rlocs)
lMap = {}
for l, p in zip(rlocs, records):
lMap[l] = p
candidates = None
with self.peersLock:
candidates = self.seekCandidates[:]
if candidates is None:
return
candidates.remove(self.loc)
#print("canidates",candidates)
if len(candidates) == 0:
return #im alone, no backups to send
for loc in rlocs:
l = lMap[loc]
bestLoc = space.getClosest(loc, candidates)
peer = self.locPeerDict[bestLoc]
value = self.database.get(l)
#print(loc,l,peer)
try:
self.network.store(
self.key, peer, l, value
) #this backs up existing values, and stores old values on new nodes.
#print("%s is backed up to %s"%(l,peer))
except Exception as e:
print(e)
continue
def seek(self,key):
"""
Answers the question: of the nodes I know, which is the closest to key?
Key is some key we are looking for.
Essentially, seek(key) is a single step of a lookup(key) operation.
Throw seek into a loop and you have iterative lookup!
"""
loc = space.idToPoint(2, key)
candidates = None
with self.peersLock:
candidates = self.seekCandidates
if len(candidates) == 0:
return self.info # as Toad would say, "I'm the best!"
bestLoc = space.getClosest(loc, candidates)
peer = self.locPeerDict[bestLoc]
return peer
def seek(self, key):
"""
Answers the question: of the nodes I know, which is the closest to key?
Key is some key we are looking for.
Essentially, seek(key) is a single step of a lookup(key) operation.
Throw seek into a loop and you have iterative lookup!
"""
loc = space.idToPoint(2, key)
candidates = None
with self.peersLock:
candidates = self.seekCandidates
if len(candidates) == 0:
return self.info # as Toad would say, "I'm the best!"
bestLoc = space.getClosest(loc, candidates)
peer = self.locPeerDict[bestLoc]
return peer
def __init__(self, peerInfo, key):
"""Initializes the node with a PeerInfo object"""
global client
self.network = None
self.database = None
self.key = key
self.shortPeers = []
self.longPeers = []
self.seekCandidates = []
self.notifiedMe = []
self.locPeerDict = {}
self.info = peerInfo
self.finger = 0
if peerInfo.loc is None:
self.loc = space.idToPoint(2, self.info.id)
self.info.loc = self.loc
else:
self.loc = peerInfo.loc
self.janitorThread = None
self.peersLock = threading.Lock()
self.notifiedLock = threading.Lock()
client = clientlib.UrDHTClient("UrDHT",[self.info.jsonify()])
def __init__(self, peerInfo, key):
"""Initializes the node with a PeerInfo object"""
global client
self.network = None
self.database = None
self.key = key
self.shortPeers = []
self.longPeers = []
self.seekCandidates = []
self.notifiedMe = []
self.locPeerDict = {}
self.info = peerInfo
self.finger = 0
if peerInfo.loc is None:
self.loc = space.idToPoint(2, self.info.id)
self.info.loc = self.loc
else:
self.loc = peerInfo.loc
self.janitorThread = None
self.peersLock = threading.Lock()
self.notifiedLock = threading.Lock()
client = clientlib.UrDHTClient("UrDHT", [self.info.jsonify()])
def run(self):
"""
Starts the thread
Needs to be split into more methods
"""
with self.runningLock:
peerCandidateSet = set()
while self.running:
#print("short",self.parent.shortPeers)
##print("myinfo",self.parent.info)
##print("Worker Tick Start")
#"Notify all my short peers"
with self.parent.peersLock:
##print("got peer lock")
peerCandidateSet.update( set(self.parent.shortPeers[:]+self.parent.longPeers[:]))
#print(peerCandidateSet)
peerCandidateSet = set(filter(self.parent.info.__ne__, peerCandidateSet)) #everyone who is not me
assert(self.parent.info not in peerCandidateSet) #everyone who is not me
##print("thinking")
#"Re-evaluate my peerlist"
with self.parent.notifiedLock: #functionize into handleNotifies
peerCandidateSet.update(set(self.parent.notifiedMe))
def pingCheck(p):
if not self.parent.network.ping(self.parent.key,p) == True:
peerCandidateSet.remove(p)
#print("Ping Failed",p)
threads = Threadpool(10)
for x in threads.map(pingCheck,set(peerCandidateSet)):
pass
points = []
locDict = {}
##print(peers_2_keep)
for p in set(peerCandidateSet):
l = space.idToPoint(2,p.id)
points.append(l)
locDict[l] = p
locDict[self.parent.loc] = self.parent.info
newShortLocsList = space.getDelaunayPeers(points,self.parent.loc)
newShortPeersList = [locDict[x] for x in newShortLocsList]
leftoversList = list(peerCandidateSet-set(newShortPeersList))
if len(newShortPeersList)<MIN_SHORTPEERS and len(leftoversList)>0:
leftoverLocsList = list(set(points)-set(newShortLocsList))
sortedLeftoverLocsList = sorted(leftoverLocsList)
needed = min((len(leftoversList),MIN_SHORTPEERS-len(newShortPeersList)))
newShortPeerLocsList = sortedLeftoverLocsList[:needed]
newShortPeersList += [locDict[x] for x in newShortPeerLocsList]
if needed < len(leftoversList):
leftoversList = [locDict[x] for x in sortedLeftoverLocsList[needed:]]
peerCandidateSet = set()
def notifyAndGet(p):
try:
self.parent.network.notify(self.parent.key,p,self.parent.info)
newpeers = self.parent.network.getPeers(self.parent.key,p)
peerCandidateSet.update(set(newpeers))
except DialFailed:
#print("DIAL FAILED",p)
with self.parent.peersLock:
if p in self.parent.shortPeers:
self.parent.shortPeers.remove(p)
if p in self.parent.longPeers:
self.parent.longPeers.remove(p)
#with self.parent.notifiedLock:
# self.parent.notifiedMe = []
threads = Threadpool(10)
#print("about to map")
for x in threads.map(notifyAndGet,set(newShortPeersList)):
pass
#print("done mapping")
trigger_change = False
leftoversList = self.parent.longPeerSelection(leftoversList)
with self.parent.peersLock:
if self.parent.shortPeers != newShortPeersList:
trigger_change = True #consider reducing this to a copy, and compare outside the lock
self.parent.shortPeers = newShortPeersList
self.parent.longPeers = leftoversList
self.parent.seekCandidates = points + [self.parent.loc]
self.parent.locPeerDict = locDict
with self.parent.notifiedLock:
self.parent.notifiedMe = []
if trigger_change or random.random()<0.0:
self.parent.onResponsibilityChange()
time.sleep(MAINTENANCE_SLEEP_PERIOD)
def run(self):
"""
Starts the thread
Needs to be split into more methods
"""
with self.runningLock:
peerCandidateSet = set()
while self.running:
#print("short",self.parent.shortPeers)
##print("myinfo",self.parent.info)
##print("Worker Tick Start")
#"Notify all my short peers"
with self.parent.peersLock:
##print("got peer lock")
peerCandidateSet.update(
set(self.parent.shortPeers[:] +
self.parent.longPeers[:]))
#print(peerCandidateSet)
peerCandidateSet = set(
filter(self.parent.info.__ne__,
peerCandidateSet)) #everyone who is not me
assert (self.parent.info
not in peerCandidateSet) #everyone who is not me
##print("thinking")
#"Re-evaluate my peerlist"
with self.parent.notifiedLock: #functionize into handleNotifies
peerCandidateSet.update(set(self.parent.notifiedMe))
def pingCheck(p):
if not self.parent.network.ping(self.parent.key,
p) == True:
peerCandidateSet.remove(p)
#print("Ping Failed",p)
threads = Threadpool(10)
for x in threads.map(pingCheck, set(peerCandidateSet)):
pass
points = []
locDict = {}
##print(peers_2_keep)
for p in set(peerCandidateSet):
l = space.idToPoint(2, p.id)
points.append(l)
locDict[l] = p
locDict[self.parent.loc] = self.parent.info
newShortLocsList = space.getDelaunayPeers(
points, self.parent.loc)
newShortPeersList = [locDict[x] for x in newShortLocsList]
leftoversList = list(peerCandidateSet - set(newShortPeersList))
if len(newShortPeersList) < MIN_SHORTPEERS and len(
leftoversList) > 0:
leftoverLocsList = list(
set(points) - set(newShortLocsList))
sortedLeftoverLocsList = sorted(leftoverLocsList)
needed = min((len(leftoversList),
MIN_SHORTPEERS - len(newShortPeersList)))
newShortPeerLocsList = sortedLeftoverLocsList[:needed]
newShortPeersList += [
locDict[x] for x in newShortPeerLocsList
]
if needed < len(leftoversList):
leftoversList = [
locDict[x] for x in sortedLeftoverLocsList[needed:]
]
peerCandidateSet = set()
def notifyAndGet(p):
try:
self.parent.network.notify(self.parent.key, p,
self.parent.info)
newpeers = self.parent.network.getPeers(
self.parent.key, p)
peerCandidateSet.update(set(newpeers))
except DialFailed:
#print("DIAL FAILED",p)
with self.parent.peersLock:
if p in self.parent.shortPeers:
self.parent.shortPeers.remove(p)
if p in self.parent.longPeers:
self.parent.longPeers.remove(p)
#with self.parent.notifiedLock:
# self.parent.notifiedMe = []
threads = Threadpool(10)
#print("about to map")
for x in threads.map(notifyAndGet, set(newShortPeersList)):
pass
#print("done mapping")
trigger_change = False
leftoversList = self.parent.longPeerSelection(leftoversList)
with self.parent.peersLock:
if self.parent.shortPeers != newShortPeersList:
trigger_change = True #consider reducing this to a copy, and compare outside the lock
self.parent.shortPeers = newShortPeersList
self.parent.longPeers = leftoversList
self.parent.seekCandidates = points + [self.parent.loc]
self.parent.locPeerDict = locDict
with self.parent.notifiedLock:
self.parent.notifiedMe = []
if trigger_change or random.random() < 0.0:
self.parent.onResponsibilityChange()
time.sleep(MAINTENANCE_SLEEP_PERIOD)