Beispiel #1
0
    def onResponsibilityChange(self):
        #print("BACKUP IS HAPPENING")
        records = list(filter(self.doIOwn, self.database.getRecords()))
        ##print(records)
        rlocs = list(map(lambda x: space.idToPoint(2, x), records))
        ##print(rlocs)
        lMap = {}
        for l,p in zip(rlocs,records):
            lMap[l]=p
        candidates = None
        with self.peersLock:
            candidates = self.seekCandidates[:]
        if candidates is None:
            return
        candidates.remove(self.loc)
        #print("canidates",candidates)
        if len(candidates) == 0:
            return #im alone, no backups to send
        for loc in rlocs:

            l = lMap[loc]
            bestLoc = space.getClosest(loc, candidates)
            peer = self.locPeerDict[bestLoc]
            value = self.database.get(l)
            #print(loc,l,peer)
            try:
                self.network.store(self.key,peer,l,value)#this backs up existing values, and stores old values on new nodes.
                #print("%s is backed up to %s"%(l,peer))
            except Exception as e:
                print(e)
                continue
Beispiel #2
0
    def onResponsibilityChange(self):
        #print("BACKUP IS HAPPENING")
        records = list(filter(self.doIOwn, self.database.getRecords()))
        ##print(records)
        rlocs = list(map(lambda x: space.idToPoint(2, x), records))
        ##print(rlocs)
        lMap = {}
        for l, p in zip(rlocs, records):
            lMap[l] = p
        candidates = None
        with self.peersLock:
            candidates = self.seekCandidates[:]
        if candidates is None:
            return
        candidates.remove(self.loc)
        #print("canidates",candidates)
        if len(candidates) == 0:
            return  #im alone, no backups to send
        for loc in rlocs:

            l = lMap[loc]
            bestLoc = space.getClosest(loc, candidates)
            peer = self.locPeerDict[bestLoc]
            value = self.database.get(l)
            #print(loc,l,peer)
            try:
                self.network.store(
                    self.key, peer, l, value
                )  #this backs up existing values, and stores old values on new nodes.
                #print("%s is backed up to %s"%(l,peer))
            except Exception as e:
                print(e)
                continue
Beispiel #3
0
    def seek(self,key):
        """
        Answers the question: of the nodes I know, which is the closest to key?
        Key is some key we are looking for.

        Essentially, seek(key) is a single step of a lookup(key) operation.
        Throw seek into a loop and you have iterative lookup!
        """
        loc = space.idToPoint(2, key)
        candidates = None
        with self.peersLock:
            candidates = self.seekCandidates
        if len(candidates) == 0:
            return self.info  # as Toad would say, "I'm the best!"
        bestLoc = space.getClosest(loc, candidates)
        peer = self.locPeerDict[bestLoc]
        return peer
Beispiel #4
0
    def seek(self, key):
        """
        Answers the question: of the nodes I know, which is the closest to key?
        Key is some key we are looking for.

        Essentially, seek(key) is a single step of a lookup(key) operation.
        Throw seek into a loop and you have iterative lookup!
        """
        loc = space.idToPoint(2, key)
        candidates = None
        with self.peersLock:
            candidates = self.seekCandidates
        if len(candidates) == 0:
            return self.info  # as Toad would say, "I'm the best!"
        bestLoc = space.getClosest(loc, candidates)
        peer = self.locPeerDict[bestLoc]
        return peer
Beispiel #5
0
 def __init__(self, peerInfo, key):
     """Initializes the node with a PeerInfo object"""
     global client
     self.network = None
     self.database = None
     self.key = key
     self.shortPeers = []
     self.longPeers = []
     self.seekCandidates = []
     self.notifiedMe = []
     self.locPeerDict = {}
     self.info = peerInfo
     self.finger = 0
     if peerInfo.loc is None:
         self.loc = space.idToPoint(2, self.info.id)
         self.info.loc = self.loc
     else:
         self.loc = peerInfo.loc
     self.janitorThread = None
     self.peersLock = threading.Lock()
     self.notifiedLock = threading.Lock()
     client = clientlib.UrDHTClient("UrDHT",[self.info.jsonify()])
Beispiel #6
0
 def __init__(self, peerInfo, key):
     """Initializes the node with a PeerInfo object"""
     global client
     self.network = None
     self.database = None
     self.key = key
     self.shortPeers = []
     self.longPeers = []
     self.seekCandidates = []
     self.notifiedMe = []
     self.locPeerDict = {}
     self.info = peerInfo
     self.finger = 0
     if peerInfo.loc is None:
         self.loc = space.idToPoint(2, self.info.id)
         self.info.loc = self.loc
     else:
         self.loc = peerInfo.loc
     self.janitorThread = None
     self.peersLock = threading.Lock()
     self.notifiedLock = threading.Lock()
     client = clientlib.UrDHTClient("UrDHT", [self.info.jsonify()])
Beispiel #7
0
    def run(self):
        """
        Starts the thread
        Needs to be split into more methods
        """
        with self.runningLock:
            peerCandidateSet = set()
            while self.running:
                #print("short",self.parent.shortPeers)
                ##print("myinfo",self.parent.info)
                ##print("Worker Tick Start")
                #"Notify all my short peers"
                with self.parent.peersLock:
                    ##print("got peer lock")
                    peerCandidateSet.update( set(self.parent.shortPeers[:]+self.parent.longPeers[:]))


                #print(peerCandidateSet)

                peerCandidateSet = set(filter(self.parent.info.__ne__, peerCandidateSet)) #everyone who is not me
                assert(self.parent.info not in peerCandidateSet) #everyone who is not me
                ##print("thinking")
                #"Re-evaluate my peerlist"
                with self.parent.notifiedLock:  #functionize into handleNotifies
                    peerCandidateSet.update(set(self.parent.notifiedMe))




                def pingCheck(p):
                    if not self.parent.network.ping(self.parent.key,p) == True:
                        peerCandidateSet.remove(p)
                        #print("Ping Failed",p)
                threads = Threadpool(10)
                for x in threads.map(pingCheck,set(peerCandidateSet)):
                    pass

                points = []
                locDict = {}

                ##print(peers_2_keep)
                for p in set(peerCandidateSet):
                    l = space.idToPoint(2,p.id)
                    points.append(l)
                    locDict[l] = p
                locDict[self.parent.loc] = self.parent.info
                newShortLocsList = space.getDelaunayPeers(points,self.parent.loc)
                newShortPeersList = [locDict[x] for x in newShortLocsList]
                leftoversList = list(peerCandidateSet-set(newShortPeersList))

                if len(newShortPeersList)<MIN_SHORTPEERS and len(leftoversList)>0:
                    leftoverLocsList = list(set(points)-set(newShortLocsList))
                    sortedLeftoverLocsList = sorted(leftoverLocsList)
                    needed = min((len(leftoversList),MIN_SHORTPEERS-len(newShortPeersList)))
                    newShortPeerLocsList = sortedLeftoverLocsList[:needed]
                    newShortPeersList += [locDict[x] for x in newShortPeerLocsList]
                    if needed < len(leftoversList):
                        leftoversList = [locDict[x] for x in sortedLeftoverLocsList[needed:]]







                peerCandidateSet = set()
                def notifyAndGet(p):
                    try:
                        self.parent.network.notify(self.parent.key,p,self.parent.info)
                        newpeers = self.parent.network.getPeers(self.parent.key,p)
                        peerCandidateSet.update(set(newpeers))
                    except DialFailed:
                        #print("DIAL FAILED",p)
                        with self.parent.peersLock:
                            if p in self.parent.shortPeers:
                                self.parent.shortPeers.remove(p)
                            if p in self.parent.longPeers:
                                self.parent.longPeers.remove(p)
                        #with self.parent.notifiedLock:
                        #    self.parent.notifiedMe = []

                threads = Threadpool(10)
                #print("about to map")
                for x in threads.map(notifyAndGet,set(newShortPeersList)):
                    pass
                #print("done mapping")

                trigger_change = False

                leftoversList = self.parent.longPeerSelection(leftoversList)


                with self.parent.peersLock:
                    if self.parent.shortPeers != newShortPeersList:
                        trigger_change = True #consider reducing this to a copy, and compare outside the lock
                    self.parent.shortPeers = newShortPeersList

                    self.parent.longPeers = leftoversList
                    self.parent.seekCandidates = points + [self.parent.loc]
                    self.parent.locPeerDict = locDict
                with self.parent.notifiedLock:
                    self.parent.notifiedMe = []
                if trigger_change or random.random()<0.0:
                    self.parent.onResponsibilityChange()
                time.sleep(MAINTENANCE_SLEEP_PERIOD)
Beispiel #8
0
    def run(self):
        """
        Starts the thread
        Needs to be split into more methods
        """
        with self.runningLock:
            peerCandidateSet = set()
            while self.running:
                #print("short",self.parent.shortPeers)
                ##print("myinfo",self.parent.info)
                ##print("Worker Tick Start")
                #"Notify all my short peers"
                with self.parent.peersLock:
                    ##print("got peer lock")
                    peerCandidateSet.update(
                        set(self.parent.shortPeers[:] +
                            self.parent.longPeers[:]))

                #print(peerCandidateSet)

                peerCandidateSet = set(
                    filter(self.parent.info.__ne__,
                           peerCandidateSet))  #everyone who is not me
                assert (self.parent.info
                        not in peerCandidateSet)  #everyone who is not me
                ##print("thinking")
                #"Re-evaluate my peerlist"
                with self.parent.notifiedLock:  #functionize into handleNotifies
                    peerCandidateSet.update(set(self.parent.notifiedMe))

                def pingCheck(p):
                    if not self.parent.network.ping(self.parent.key,
                                                    p) == True:
                        peerCandidateSet.remove(p)
                        #print("Ping Failed",p)

                threads = Threadpool(10)
                for x in threads.map(pingCheck, set(peerCandidateSet)):
                    pass

                points = []
                locDict = {}

                ##print(peers_2_keep)
                for p in set(peerCandidateSet):
                    l = space.idToPoint(2, p.id)
                    points.append(l)
                    locDict[l] = p
                locDict[self.parent.loc] = self.parent.info
                newShortLocsList = space.getDelaunayPeers(
                    points, self.parent.loc)
                newShortPeersList = [locDict[x] for x in newShortLocsList]
                leftoversList = list(peerCandidateSet - set(newShortPeersList))

                if len(newShortPeersList) < MIN_SHORTPEERS and len(
                        leftoversList) > 0:
                    leftoverLocsList = list(
                        set(points) - set(newShortLocsList))
                    sortedLeftoverLocsList = sorted(leftoverLocsList)
                    needed = min((len(leftoversList),
                                  MIN_SHORTPEERS - len(newShortPeersList)))
                    newShortPeerLocsList = sortedLeftoverLocsList[:needed]
                    newShortPeersList += [
                        locDict[x] for x in newShortPeerLocsList
                    ]
                    if needed < len(leftoversList):
                        leftoversList = [
                            locDict[x] for x in sortedLeftoverLocsList[needed:]
                        ]

                peerCandidateSet = set()

                def notifyAndGet(p):
                    try:
                        self.parent.network.notify(self.parent.key, p,
                                                   self.parent.info)
                        newpeers = self.parent.network.getPeers(
                            self.parent.key, p)
                        peerCandidateSet.update(set(newpeers))
                    except DialFailed:
                        #print("DIAL FAILED",p)
                        with self.parent.peersLock:
                            if p in self.parent.shortPeers:
                                self.parent.shortPeers.remove(p)
                            if p in self.parent.longPeers:
                                self.parent.longPeers.remove(p)
                        #with self.parent.notifiedLock:
                        #    self.parent.notifiedMe = []

                threads = Threadpool(10)
                #print("about to map")
                for x in threads.map(notifyAndGet, set(newShortPeersList)):
                    pass
                #print("done mapping")

                trigger_change = False

                leftoversList = self.parent.longPeerSelection(leftoversList)

                with self.parent.peersLock:
                    if self.parent.shortPeers != newShortPeersList:
                        trigger_change = True  #consider reducing this to a copy, and compare outside the lock
                    self.parent.shortPeers = newShortPeersList

                    self.parent.longPeers = leftoversList
                    self.parent.seekCandidates = points + [self.parent.loc]
                    self.parent.locPeerDict = locDict
                with self.parent.notifiedLock:
                    self.parent.notifiedMe = []
                if trigger_change or random.random() < 0.0:
                    self.parent.onResponsibilityChange()
                time.sleep(MAINTENANCE_SLEEP_PERIOD)