예제 #1
0
    def __init__(self, timer):
        self.timer = timer

        self.userNodes = []
        self.sharedStat = {}
        self.sharedStat['userNodes'] = self.userNodes
        self.sharedStat['timer'] = self.timer
        self.x, self.y = Util.loc2coor(self.latitude, self.longitude)

        self.inBuf = FQueue()
        self.outBuf = FQueue()
        self._init_follower_set()
예제 #2
0
    def __init__(self, netIn, netOut, pFail, sharedStat, id, x, y):
        self.inBuf = PQueue()
        self.outBuf = FQueue() 
        self.isWorking = True 
        self.delayRoll = [0] * self.DELAY_SLOT_NUM

        self.netIn = netIn
        self.netOut = netOut
        self.pFail = pFail
        self.userNodes = sharedStat['userNodes']
        self.timer = sharedStat['timer']
        self.msgCount = MsgCounter(self.timer)
        self.id = id
        self.x = x
        self.y = y
예제 #3
0
class Server:
    SAMPLE_RATE = 0.1

    P2P_TH = 20
   
    id = 0 
    inBuf = None
    outBuf = None

    #twitter office location, well, I don't know where its is...
    latitude = 37.782587
    longitude = -122.400595
    x = 0
    y = 0

    #10GB incoming outgoing network bandwidth
    netIn = 10 * 1024 * 1024 * 1024  
    netOut = netIn

    #keep current server in/out bandwidth
    curNetIn = 0
    curNetOut = 0

    sbNetOut = 0 #server broadcast net out

    liveProb = 0.05

    #10KB user incoming and outgoing network bandwidh
    userNetIn = 10 * 1024
    userNetOut = userNetIn
    userPFail = 0.01

    #for tree partition 
    userAngle = math.pi / 6
    angle = math.pi
    # (d, h) = (4, 5) lead to at most 340 descendant
    d = 4
    h = 5

    userNodes = None
    userNum = 0 
    userIDs = None

    coor_file = "../../data/small_coors"

    sharedStat = None
    timer = None

    #accumulated net resources
    accIn = 0
    accOut = 0

    #msg missing rate
    msgRecExpCnt = 0
    REDUNDANCY = 3

    def __init__(self, timer):
        self.timer = timer

        self.userNodes = []
        self.sharedStat = {}
        self.sharedStat['userNodes'] = self.userNodes
        self.sharedStat['timer'] = self.timer
        self.x, self.y = Util.loc2coor(self.latitude, self.longitude)

        self.inBuf = FQueue()
        self.outBuf = FQueue()
        self._init_follower_set()

    def _init_follower_set(self):
        f = open(self.coor_file, 'r')
        cnt = 0
        for line in f:
            items = line.split(',')
            latitude = float(items[0])
            longitude = float(items[1])
            x, y = Util.loc2coor(latitude, longitude)
            user = UserNode(self.userNetIn, self.userNetOut, self.userPFail, self.sharedStat, cnt, x, y)
            self.userNodes.append(user)
            cnt += 1
        f.close()
        self.userNum = cnt
        print "**"
        self.userIDs = range(self.userNum)
        self.id = cnt

    def get_user_nodes(self):
        return self.userNodes        

    #unified interface for the scheduler to call
    def receive(self):
        """
        1. consider the network constraints
        2. read from inBuf, compute the DHTree and put msgs into outBuf
        """
        self.accIn += self.netIn
        self.curNetIn = 0
        self.sbNetOut = 0
        data = self.get_from_in_buf()
        self.msgRecExpCnt = 0
        while data:
            timestamp, msgID, folNum, msgLen = data
            folNum = int(math.ceil(self.liveProb * folNum))
            if folNum <= 0:
                data = self.get_from_in_buf()
                continue
            #we do not need to count the incoming traffic, as it gonna be the same with or without Centuar
            uniq, peerIDs = self._get_follower_ids(folNum)
            senderID = random.choice(peerIDs)
            if folNum < self.P2P_TH:
                self.userNodes[senderID].store_msg_peer(msgID, peerIDs)
            else:
                gossipNum = int(math.ceil(math.log(folNum)))
                for peer in peerIDs:
                    self.notify_msg_peer(peer, msgID, peerIDs, gossipNum)

            self.userNodes[senderID].put_to_in_buf(self.timer.cur_time(), timestamp, msgID,  msgLen)

            self.msgRecExpCnt += uniq
            data = self.get_from_in_buf()
 
    

    """
    inBuf data fields:
    1. follower num
    2. msg len
    """
    def get_from_in_buf(self):
        # we consider the msg as the entire json string
        data = self.inBuf.peek()
        if not data:
            #inBuf is empty
            return None
        timestamp, msgID, folNum, msgLen = data
        if msgLen > self.accIn:
            return None
        else:
            self.accIn -= msgLen
            return self.inBuf.pop()

    #folNum is used to determin the number of subscribers for this msg
    def put_to_in_buf(self, timestamp, msgID, folNum, msgLen):
        self.inBuf.push((timestamp, msgID, folNum, msgLen)) 

    def get_msg_rec_exp_cnt(self):
        return self.msgRecExpCnt

    def notify_msg_peer(self, userID, msgID, folSet, size):
        #pick size users form folSet, and send the (msgID, list) to userID
        peerList = random.sample(folSet, size)
        self.userNodes[userID].store_msg_peer(msgID, peerList) 

    def _get_follower_ids(self, num):
        if num < self.userNum:
            return (num, random.sample(self.userIDs, num))
        folSet = []
        uniqueSet = set()
        while num > 0:
            id = random.randint(0, self.userNum - 1)
            uniqueSet.add(id)
            folSet.append(id)
            num -= 1
        return (len(uniqueSet), folSet)
예제 #4
0
class Server:
    SAMPLE_RATE = 0.5
   
    id = 0 
    inBuf = None
    outBuf = None

    #twitter office location, well, I don't know where its is...
    latitude = 37.782587
    longitude = -122.400595
    x = 0
    y = 0

    #10GB incoming outgoing network bandwidth
    netIn = 10 * 1024 * 1024 * 1024  
    netOut = netIn

    #keep current server in/out bandwidth
    curNetIn = 0
    curNetOut = 0

    sbNetOut = 0 #server broadcast net out

    liveProb = 0.05

    #10KB user incoming and outgoing network bandwidh
    userNetIn = 10 * 1024
    userNetOut = userNetIn
    userPFail = 0.05

    #for tree partition 
    userAngle = math.pi / 6
    angle = math.pi
    # (d, h) = (4, 5) lead to at most 340 descendant
    d = 2
    h = 10

    userNodes = None
    userNum = 0 

    coor_file = "../../data/small_coors"

    sharedStat = None
    timer = None

    #accumulated net resources
    accIn = 0
    accOut = 0

    #msg missing rate
    msgRecExpCnt = 0
    REDUNDANCY = 3

    def __init__(self, timer):
        self.timer = timer

        self.userNodes = []
        self.sharedStat = {}
        self.sharedStat['userNodes'] = self.userNodes
        self.sharedStat['timer'] = self.timer
        self.x, self.y = Util.loc2coor(self.latitude, self.longitude)

        self.inBuf = FQueue()
        self.outBuf = FQueue()
        self._init_follower_set()

    def _init_follower_set(self):
        f = open(self.coor_file, 'r')
        cnt = 0
        for line in f:
            items = line.split(',')
            latitude = float(items[0])
            longitude = float(items[1])
            x, y = Util.loc2coor(latitude, longitude)
            user = UserNode(self.userNetIn, self.userNetOut, self.userPFail, self.sharedStat, cnt, x, y)
            self.userNodes.append(user)
            cnt += 1
        f.close()
        self.userNum = cnt
        self.id = cnt

    def get_user_nodes(self):
        return self.userNodes        

    #unified interface for the scheduler to call
    def receive(self):
        """
        1. consider the network constraints
        2. read from inBuf, compute the DHTree and put msgs into outBuf
        """
        self.accIn += self.netIn
        self.curNetIn = 0
        self.sbNetOut = 0
        data = self.get_from_in_buf()
        self.msgRecExpCnt = 0
        while data:
            timestamp, msgID, folNum, msgLen = data
            folNum = math.ceil(self.liveProb * folNum)
            #we do not need to count the incoming traffic, as it gonna be the same with or without Centuar
            uniqCnt, folSet = self._get_followers(folNum)
            self.msgRecExpCnt += uniqCnt
            folSet.insert(0, [self.id, self.x, self.y])
            self.sbNetOut += (folNum * (Vertex.NODE_SIZE_IN_MEM + msgLen))
            for i in range(self.REDUNDANCY):
                tree = DHTree(folSet)
                r = tree.get_tree(self.userAngle, self.angle, self.d, self.h)
                for u in r.cList:
                    self.put_to_out_buf(timestamp, msgID, u, msgLen)
            data = self.get_from_in_buf()
 
    
    def send(self):
        """
        1. consider the network constraints and send out the msgs
        2. for each tweet, construct the follower set
        3. generate the multicasting tree
        4. do the redundency
        """
        self.accOut += self.netOut
        self.curNetOut = 0
        data = self.get_from_out_buf()
        while data:
            timestamp, msgID, u, msgLen = data
            x1 = self.userNodes[u.id].x
            y1 = self.userNodes[u.id].y
            delay = Util.delay(self.x, self.y, x1, y1)
            #net = Util.net(self.x, self.y, x1, y1)
            #delay += ((u.subTreeSize + msgLen ) / net )
            self.userNodes[u.id].put_to_in_buf(self.timer.cur_time() + delay, timestamp, msgID, u, msgLen)
            data = self.get_from_out_buf()

    """
    inBuf data fields:
    1. follower num
    2. msg len
    """
    def get_from_in_buf(self):
        # we consider the msg as the entire json string
        data = self.inBuf.peek()
        if not data:
            #inBuf is empty
            return None
        timestamp, msgID, folNum, msgLen = data
        if msgLen > self.accIn:
            return None
        else:
            self.accIn -= msgLen
            self.curNetIn += msgLen
            return self.inBuf.pop()

    #folNum is used to determin the number of subscribers for this msg
    def put_to_in_buf(self, timestamp, msgID, folNum, msgLen):
        self.inBuf.push((timestamp, msgID, folNum, msgLen)) 

    def get_from_out_buf(self):
        data = self.outBuf.peek()
        if not data:
            #outBuf is empty
            return None
        timestamp, msgID, u, msgLen = data
        #id takes 8 bytes, in real case it might need more: ip, and other info. TODO: see what is required to start an connection
        size = u.subTreeSize * u.NODE_SIZE_IN_MEM + msgLen
        if size > self.accOut:
            return None
        else:
            self.accOut -= size 
            self.curNetOut += size
            return self.outBuf.pop()

    def get_cur_net_stat(self):
        return (self.curNetIn, self.curNetOut, self.sbNetOut)

    def get_msg_rec_exp_cnt(self):
        return self.msgRecExpCnt

    def put_to_out_buf(self, timestamp, msgID, u, msgLen):
        self.outBuf.push((timestamp, msgID, u, msgLen))

    def _get_followers(self, num):
        #given the number of living followers, randomly generate the online follower set 
        folSet = []
        uniqueSet = set()
        while num > 0:
            id = random.randint(0, self.userNum - 1)
            uniqueSet.add(id)
            folSet.append([id, self.userNodes[id].x, self.userNodes[id].y])
            num -=1 
        return (len(uniqueSet), folSet)
예제 #5
0
class UserNode:

    #corresponds to the ID in the DH tree (in the real system, an IP address or P2P ID is also required.)
    id = -1
    #pointer to the array of all user nodes
    userNodes = None
    p2pHops = 0
    timer = None
    #incoming network bandwidth
    netIn = 0

    #outgoing network bandwidth
    netOut = 0

    #node failing rate
    pFail = 0.001

    #coordination
    x = 0
    y = 0

    #pending msg peer list: the size should be bounded
    msgPeerDict = None
    #delMsgSet = None
    MSG_PEER_LEN = 50
    DEL_MSG_LEN = 500

    #an array of receiving packets: priority queue with delay as weight, be careful when maintaining the timestamps
    inBuf = None
    #an array of sending packets: FIFO
    outBuf = None
    
    #accumulated incoming network resources: the user receives the packet only when the accumulated incoming resource is larger than the size of the packet
    accIn = 0
    accOut = 0

    #userNetStat
    curNetIn = 0
    curNetOut = 0
    accNetDelay = 0

    #msgDelayStat
    #DELAY_SLOT_NUM = 100
    delayIndex = 0
    maxDelay = 0
    #delayCnt = 0
    #delayRoll = None

    #msg missing rate stat
    msgCount = None
    newMsgCount = 0

    def __init__(self, netIn, netOut, pFail, sharedStat, id, x, y):
        self.inBuf = PQueue()
        self.outBuf = FQueue() 
        self.isWorking = True 
        #self.delayRoll = [0] * self.DELAY_SLOT_NUM

        self.msgPeerDict = rbtree.rbtree()
        #self.delMsgSet = rbtree.rbtree()

        self.netIn = netIn
        self.netOut = netOut
        self.pFail = pFail
        self.userNodes = sharedStat['userNodes']
        self.timer = sharedStat['timer']
        #self.msgCount = MsgCounter(self.timer)
        self.id = id
        self.x = x
        self.y = y

    def start(self, hops):
        self.p2pHops = hops

    def store_msg_peer(self, msgID, peerList):
        self.msgPeerDict[msgID] = peerList
        if len(self.msgPeerDict) > self.MSG_PEER_LEN:
            #print ("too many entries in msgPeerDict")
            self.msgPeerDict.pop()
        
        while math.fabs(self.msgPeerDict.min()) + 10000 < msgID:
            #print ("poped", self.msgPeerDict.min())
            self.msgPeerDict.pop()

    def del_msg(self, msgID):
        if self.msgPeerDict.has_key(msgID):
            del self.msgPeerDict[msgID]
            #self.delMsgSet[msgID] = None
            #if len(self.delMsgSet) > self.DEL_MSG_LEN:
            #    self.delMsgSet.pop()

    def report_delay(self, delay):
        if self.maxDelay < delay:
            self.maxDelay = delay

    def get_delay(self):
        tmpDelay = self.maxDelay
        self.maxDelay = 0
        return tmpDelay

    def receive(self):
        """
        0. inc accIn budget
        1. read from inBuf (consider both cpu and network constraints)
        2. append subtrees to the outBuf
        """
        self.accIn += self.netIn
        self.curNetIn = 0        
        self.accNetDelay = max(0, self.accNetDelay - 1)
        self.newMsgCount = 0
        self.maxDelay = 0

        data = self.get_from_in_buf()
        while data:
            prevDelay, (timestamp, msgID, msgLen) = data
            self.report_delay(prevDelay - timestamp)
            #if self.msgCount.report_receive(msgID):
            #    self.newMsgCount += 1
            
            #if not has_key, it means that the user has already received the msg before, or the msg is too old in which case, the entry has been deleted


            if self.msgPeerDict.has_key(msgID):
                self.newMsgCount += 1
                peerList = self.msgPeerDict[msgID]
                #self.del_msg(msgID)
                del self.msgPeerDict[msgID]
                self.put_to_out_buf(prevDelay, timestamp, msgID, peerList, msgLen)
            elif msgID < 0 and self.msgPeerDict.has_key(-msgID):
                self.newMsgCount += 1
                peerList = self.msgPeerDict[-msgID]
                del self.msgPeerDict[-msgID]
                self.put_to_out_buf(prevDelay, timestamp, msgID, peerList, msgLen)
            elif msgID < 0:
                self.newMsgCount += 1
            data = self.get_from_in_buf()
        #self.msgCount.remove_deprecated()

    def send(self):
        """
        0. inc accOut budget
        1. read from outBuf (consider both cpu and network constraints)
        2. append to receivers' inBuf
        """
        self.accOut += self.netOut
        self.curNetOut = 0

        data = self.get_from_out_buf()
        while data:
            prevDelay, timestamp, msgID, peerList, msgLen = data
            #Arr... there is some redundancy here, both self.userNodes[u.id].x and u.x refers to the same thing
            for peerID in peerList:
                x1 = self.userNodes[peerID].x
                y1 = self.userNodes[peerID].y
                delay = self.p2pHops * Util.delay(self.x, self.y, x1, y1)
                net = Util.net(self.x, self.y, x1, y1)
                delay += (msgLen  / net )
                #print prevDelay + delay
                self.userNodes[peerID].put_to_in_buf(prevDelay + delay, timestamp, msgID, msgLen)
            del peerList 
            data = self.get_from_out_buf()
            

    #should be called by function receive
    def get_from_in_buf(self):
        """
        get one package from the inBuf if both cpu and network contraints allows
        """
        data = self.inBuf.peek()
        if not data:
            return None
        #the data should be (subtree, msg) pair
        delay, (timestamp, msgID, msgLen) = data
        size = msgLen
        if self.timer.cur_time() > delay and size <= self.accIn:
            self.accIn -= size
            self.curNetIn += size
            self.inBuf.pop()
            return data
        else:
            return None

    #should be called by function send of other nodes
    def put_to_in_buf(self, delay, timestamp, msgID, msgLen):
        """
        put one package to the inBuf
        """
        self.accNetDelay += (float(msgLen ) / self.netIn)
        delay += self.accNetDelay
        self.inBuf.push(delay, (timestamp, msgID, msgLen))

    #should be called by function receive
    def put_to_out_buf(self, prevDelay, timestamp, msgID, peerList, msgLen):
        self.outBuf.push((prevDelay, timestamp, msgID, peerList, msgLen))

    #should be called by function send
    def get_from_out_buf(self):
        data = self.outBuf.peek()
        if not data:
            return None
        prevDelay, timestamp, msgID, peerList, msgLen = data
        size = len(peerList) * msgLen
        if size <= self.accOut:
            self.accOut -= size
            self.curNetOut += size
            self.outBuf.pop()
            return data
        else:
            return None

    def is_done(self):
        if self.inBuf.is_empty() and self.outBuf.is_empty():
            return True
        else:
            return False

    def rand_fail(self):
        """
        1. when failed drop all msgs in both inBuf and outBuf
        """
        if random.random() < self.pFail:
            self.inBuf.clear()
            self.outBuf.clear()  
            self.msgPeerDict.clear()
            #self.delMsgSet.clear() 

    def get_cur_net_stat(self):
        return (self.curNetIn, self.curNetOut) 

    def get_new_msg_count(self):
        return self.newMsgCount
예제 #6
0
class UserNode:

    #corresponds to the ID in the DH tree (in the real system, an IP address or P2P ID is also required.)
    id = -1
    #pointer to the array of all user nodes
    userNodes = None
    timer = None
    #incoming network bandwidth
    netIn = 0

    #outgoing network bandwidth
    netOut = 0

    #node failing rate
    pFail = 0

    #coordination
    x = 0
    y = 0

    #an array of receiving packets: priority queue with delay as weight, be careful when maintaining the timestamps
    inBuf = None
    #an array of sending packets: FIFO
    outBuf = None
    
    #accumulated incoming network resources: the user receives the packet only when the accumulated incoming resource is larger than the size of the packet
    accIn = 0
    accOut = 0

    #userNetStat
    curNetIn = 0
    curNetOut = 0
    accNetDelay = 0

    #msgDelayStat
    DELAY_SLOT_NUM = 10
    delayIndex = 0
    delayCnt = 0
    delayRoll = None
    maxDelay = 0

    #msg missing rate stat
    msgCount = None
    newMsgCount = 0

    def __init__(self, netIn, netOut, pFail, sharedStat, id, x, y):
        self.inBuf = PQueue()
        self.outBuf = FQueue() 
        self.isWorking = True 
        self.delayRoll = [0] * self.DELAY_SLOT_NUM

        self.netIn = netIn
        self.netOut = netOut
        self.pFail = pFail
        self.userNodes = sharedStat['userNodes']
        self.timer = sharedStat['timer']
        self.msgCount = MsgCounter(self.timer)
        self.id = id
        self.x = x
        self.y = y

    def report_delay(self, delay):
        #self.delayCnt = min(self.delayCnt + 1, self.DELAY_SLOT_NUM)
        #print ("user delay: ", delay)
        #self.delayRoll[self.delayIndex] = delay
        #self.delayIndex = (self.delayIndex + 1) % self.DELAY_SLOT_NUM
        if self.maxDelay < delay:
            self.maxDelay = delay

    def get_delay(self):
        tmpDelay = self.maxDelay
        self.maxDelay = 0
        return tmpDelay

    def receive(self):
        """
        0. inc accIn budget
        1. read from inBuf (consider both cpu and network constraints)
        2. append subtrees to the outBuf
        """
        self.accIn += self.netIn
        self.curNetIn = 0        
        self.accNetDelay = max(0, self.accNetDelay - 1)
        self.newMsgCount = 0

        data = self.get_from_in_buf()
        while data:
            prevDelay, (timestamp, msgID, u, msgLen) = data
            self.report_delay(prevDelay - timestamp)
            if self.msgCount.report_receive(msgID):
                self.newMsgCount += 1
            #v is the reference to the Vertex object in the DHTree
            for v in u.cList:
                self.put_to_out_buf(prevDelay, timestamp, msgID, v, msgLen) 
            data = self.get_from_in_buf()
        self.msgCount.remove_deprecated()

    def send(self):
        """
        0. inc accOut budget
        1. read from outBuf (consider both cpu and network constraints)
        2. append to receivers' inBuf
        """
        self.accOut += self.netOut
        self.curNetOut = 0

        data = self.get_from_out_buf()
        while data:
            prevDelay, timestamp, msgID, u, msgLen = data
            #Arr... there is some redundancy here, both self.userNodes[u.id].x and u.x refers to the same thing
            x1 = self.userNodes[u.id].x
            y1 = self.userNodes[u.id].y
            delay = Util.delay(self.x, self.y, x1, y1)
            net = Util.net(self.x, self.y, x1, y1)
            delay += ((u.subTreeSize + msgLen ) / net )
            #print ("send delay", delay)
            #print (self.x, self.y, x1, y1, delay)
            #ASSERTION: cur_time() should be in seconds (float)
            self.userNodes[u.id].put_to_in_buf(prevDelay + delay, timestamp, msgID, u, msgLen) 
            data = self.get_from_out_buf()
            

    #should be called by function receive
    def get_from_in_buf(self):
        """
        get one package from the inBuf if both cpu and network contraints allows
        """
        data = self.inBuf.peek()
        if not data:
            return None
        #the data should be (subtree, msg) pair
        delay, (timestamp, msgID, u, msgLen) = data
        size = u.subTreeSize * u.NODE_SIZE_IN_MEM + msgLen
        if self.timer.cur_time() > delay and size <= self.accIn:
            self.accIn -= size
            self.curNetIn += size
            self.inBuf.pop()
            return data
        else:
            return None

    #should be called by function send of other nodes
    def put_to_in_buf(self, delay, timestamp, msgID, subtree, msgLen):
        """
        put one package to the inBuf
        """
        self.accNetDelay += (float(subtree.subTreeSize * subtree.NODE_SIZE_IN_MEM  + msgLen ) / self.netIn)
        delay += self.accNetDelay
        self.inBuf.push(delay, (timestamp, msgID, subtree, msgLen))

    #should be called by function receive
    def put_to_out_buf(self, prevDelay, timestamp, msgID, subtree, msgLen):
        self.outBuf.push((prevDelay, timestamp, msgID, subtree, msgLen))

    #should be called by function send
    def get_from_out_buf(self):
        data = self.outBuf.peek()
        if not data:
            return None
        prevDelay, timestamp, msgID, u, msgLen = data
        size = u.subTreeSize * u.NODE_SIZE_IN_MEM + msgLen
        if size <= self.accOut:
            self.accOut -= size
            self.curNetOut += size
            self.outBuf.pop()
            return data
        else:
            return None

    def is_done(self):
        if self.inBuf.is_empty() and self.outBuf.is_empty():
            return True
        else:
            return False

    def rand_fail(self):
        """
        1. when failed drop all msgs in both inBuf and outBuf
        """
        if random.random() < self.pFail:
            self.inBuf.clear()
            self.outBuf.clear()   

    def get_cur_net_stat(self):
        return (self.curNetIn, self.curNetOut) 

    def get_new_msg_count(self):
        return self.newMsgCount