def stop(self):
self.log.info('producer scheduler is stopping')
try:
self.loopingCall.stop()
except:
pass
self.buffer = Buffer(buffersize=self.stream.scheduler['bufsize'],log=self.log)
self.blockCache = {}
def initScheduler(self):
self.log.info('initing producer scheduler')
self.registerMessages()
self.loopingCall = task.LoopingCall(self.shift)
self.blocksSec = self.stream.scheduler['blocksec']
self.frequency = 1.0/self.stream.scheduler['blocksec']
self.buffer = Buffer(buffersize=self.stream.scheduler['bufsize'],log=self.log)
self.blockCache = {}
def initScheduler(self):
self.log.info('initing scheduler')
self.running = False
self.registerMessages()
self.loopingCall = task.LoopingCall(self.shift)
self.reqInterval = self.stream.scheduler['reqInt']
self.frequency = 1.0 / self.stream.scheduler['blocksec']
self.buffer = Buffer(buffersize=self.stream.scheduler['bufsize'],
log=self.log)
def initScheduler(self):
self.log.info('initing scheduler')
self.running = False
self.registerMessages()
self.loopingCall = task.LoopingCall(self.shift)
self.reqInterval = self.stream.scheduler['reqInt']
self.frequency = 1.0 / self.stream.scheduler['blocksec']
self.buffer = Buffer(buffersize=self.stream.scheduler['bufsize'],
log=self.log)
self.countHit = 0
self.countMiss = 0
self.startTime = 0
self.idleTime = 0
self.lastIdleTime = 0
self.lastReqCheck = 0
self.requestFrequency = self.frequency * self.reqInterval
def _decode(self, obj, ctx):
return Buffer(buffersize=obj.buffersize,
lpb=obj.lpb,
buffer=obj.buffer)
"subswappeerlistmessage": SubSwapPeerListMessage,
"subspeerlistmessage": SubSPeerListMessage,
'subsatelitemessage': SubSateliteMessage,
"sublockmessage": SubLockMessage,
}
if __name__ == "__main__":
from p2ner.base.Peer import Peer
p = Peer("127.0.0.1", 50000, 50003)
a = Container(message=[p], streamid=1, header=Container(ack=True, code=6))
from p2ner.base.Buffer import Buffer
b = BufferMessage.build(
Container(header=Container(code=1, ack=False),
streamid=22,
buffer=Buffer(lpb=15),
request=[3, 6, 8]))
print len(b)
print BufferMessage.parse(b)
b = BufferMessage.build(
Container(header=Container(code=1, ack=False),
streamid=22,
buffer=Buffer(lpb=15),
request=None))
print len(b)
print BufferMessage.parse(b)
exit(0)
from p2ner.base.Stream import Stream
s = '\x0b\x00\x00\x00\x00\x00\x0cpopipopi\x00\x00\x00\x00\x00\x00\x00\x9c?\x14\x07desc\x00\x08\x03'
c = StreamMessage.parse(s)
print c
class PullClient(Scheduler):
def registerMessages(self):
self.messages = []
self.messages.append(BufferMessage())
self.messages.append(LPBMessage())
self.messages.append(RetransmitMessage())
self.blocks = []
self.blocks.append(Block())
def initScheduler(self):
self.log.info('initing scheduler')
self.running = False
self.registerMessages()
self.loopingCall = task.LoopingCall(self.shift)
self.reqInterval = self.stream.scheduler['reqInt']
self.frequency = 1.0 / self.stream.scheduler['blocksec']
self.buffer = Buffer(buffersize=self.stream.scheduler['bufsize'],
log=self.log)
self.countHit = 0
self.countMiss = 0
self.startTime = 0
self.idleTime = 0
self.lastIdleTime = 0
self.lastReqCheck = 0
self.requestFrequency = self.frequency * self.reqInterval
def errback(self, failure):
return failure
def produceBlock(self):
# print "IN SCHEDULER PRODUCEBLOCK"
#self.log.debug('trying to produce block')
self.running = True
d = deferToThread(self.getRequestedBID)
d.addCallback(self.sendBlock)
d.addErrback(self.errback)
return d
def sendBlock(self, req):
# print "in SCHEDULER sendblock"
if not req:
self.running = False
if not self.lastIdleTime:
self.lastIdleTime = time()
# print "NOT SENDING"
#self.log.warning('no blocks to send. stopping scheduler')
return None
if self.lastIdleTime:
tempidle = time() - self.lastIdleTime
self.idleTime += tempidle
#self.log.debug('idle for %f',tempidle)
self.lastIdleTime = 0
self.running = True
bid, peer = req
self.log.debug('sending block %d to %s', bid, peer)
# print 'sending block %d to %s'%(bid,peer)
self.trafficPipe.call("sendblock", self, bid, peer)
counter(self, "blocksent")
def getRequestedBID(self):
# print "GETREQUESTEDBID"
while True:
#print self.bufferlist
peer = getMostDeprivedReq(self.bufferlist, self.buffer)
if peer is None:
self.running = False
#print "STOP SERVING\n\n"
#self.log.warning('no deprived peer')
self.lastReqCheck = time()
return None
#self.log.debug('requests from most deprived %s %s',peer,peer.s[self.stream.id]["request"])
bl = self.buffer.bIDListCompTrue(peer.s[self.stream.id]["request"])
bl2 = bl[:]
# print "BL:", bl
#self.log.debug('possible blocks to send %s',bl)
if len(bl) > 0:
#blockID = choice(bl)
rarest = False
try:
blockID = getRarestRequest(self.bufferlist, self.buffer,
bl)
rarest = True
except:
blockID = choice(bl)
#blockID=bl[0]
try:
peer.s[self.stream.id]["request"].remove(blockID)
peer.s[self.stream.id]["buffer"].update(blockID)
except:
self.log.error(
"problem in scheduler. Can't remove blockid %s" %
blockID)
self.log.error(
"blocks:%s,requests:%s,blocks before:%s" %
(bl, peer.s[self.stream.id]["request"], bl2))
self.log.error("happened while searching for rarest:%s" %
rarest)
continue
#print "SENDING BLOCK", blockID, peer
self.lastReqCheck = time()
return (blockID, peer)
else:
peer.s[self.stream.id]["request"] = []
def start(self):
self.startTime = time()
self.idleTime = 0
self.lastIdleTime = 0
self.countHit = 0
self.countMiss = 0
self.log.info('scheduler is starting')
self.loopingCall.start(self.frequency)
self.trafficPipe.call('resetIdleStat')
def stop(self):
self.log.info('scheduler is stopping')
#reactor.callLater(0, self.stream.stop)
try:
self.loopingCall.stop()
except:
pass
def makeRequests(self, receivingBlocks, missingBlocks, neighbours):
#print 'neighbours:',neighbours
#print "COMPUTING REQUESTS"
#print missingBlocks
#exclude receiving
# print "MISSING", missingBlocks
# print "RECEIVING", receivingBlocks
def dd(self, receivingBlocks, missingBlocks, neighbours):
if not neighbours:
return
for bid in missingBlocks:
if bid in receivingBlocks:
missingBlocks.remove(bid)
######
#TODO: manage lpb0
######
#print 'missing blocks:',missingBlocks
tmpBlocksToRequest = {}
requestableBlocks = {}
ids = {}
countIds = 0
for peer in neighbours:
if self.stream.id not in peer.s:
print 'in cotinue 1'
continue
if "buffer" not in peer.s[self.stream.id]:
print 'in continue 2'
continue
buffer = peer.s[self.stream.id]["buffer"]
# print "BUFFER", buffer
#print 'neigh buffer:',buffer
tempReq = buffer.bIDListCompTrue(missingBlocks)
tmpBlocksToRequest[peer] = tempReq
#print 'temp:',tempReq
for b in tempReq:
if b in requestableBlocks:
requestableBlocks[b].append(peer)
else:
requestableBlocks[b] = [peer]
ids[peer] = 'p' + str(countIds)
countIds += 1
if not requestableBlocks:
return {}
blocksToRequest = {}
for p in neighbours:
blocksToRequest[p] = []
reqBlockList = requestableBlocks.keys()
for b in reqBlockList:
if len(requestableBlocks[b]) == 1:
peer = requestableBlocks[b][0]
blocksToRequest[peer] += [b]
del requestableBlocks[b]
if requestableBlocks:
G = nx.DiGraph()
#first only good peers
execute = False
for b, peers in requestableBlocks.items():
G.add_edge('s', b, capacity=1)
for p in peers:
if p.reportedBW > 20:
G.add_edge(b,
ids[p],
capacity=1,
weight=len(peers))
execute = True
else:
G.add_edge(b,
ids[p],
capacity=0,
weight=len(peers))
for id in ids.values():
G.add_edge(id, 'e', capacity=1)
if execute:
try:
F = nx.min_cost_flow(G, 's', 'e')
except:
self.log.error('scheduler matching failed')
import sys
sys.exit()
for peer, id in ids.items():
reqBlocks = [
b for b in F.keys()
if b in requestableBlocks.keys()
and F[b].has_key(id) and int(F[b][id]) == 1
]
blocksToRequest[peer] += reqBlocks
for b in reqBlocks:
del requestableBlocks[b]
if requestableBlocks:
G = nx.DiGraph()
#then only bad peers
execute = False
for b, peers in requestableBlocks.items():
G.add_edge('s', b, capacity=1)
for p in peers:
if p.reportedBW <= 20:
G.add_edge(b,
ids[p],
capacity=1,
weight=len(peers))
execute = True
else:
G.add_edge(b,
ids[p],
capacity=0,
weight=len(peers))
for id in ids.values():
G.add_edge(id, 'e', capacity=1)
if execute:
try:
F = nx.min_cost_flow(G, 's', 'e')
except:
self.log.error('scheduler matching failed')
import sys
sys.exit()
for peer, id in ids.items():
reqBlocks = [
b for b in F.keys()
if b in requestableBlocks.keys()
and F[b].has_key(id) and int(F[b][id]) == 1
]
blocksToRequest[peer] += reqBlocks
for b in reqBlocks:
del requestableBlocks[b]
"""
if requestableBlocks:
G=nx.DiGraph()
#finally all
execute=False
for b,peers in requestableBlocks.items():
G.add_edge('s',b,capacity=1)
for p in peers:
G.add_edge(b,ids[p],capacity=1,weight=len(peers))
execute=True
for id in ids.values():
G.add_edge(id,'e')
if execute:
try:
F=nx.max_flow_min_cost(G,'s','e')
except:
self.log.error('scheduler matching failed')
import sys
sys.exit()
for peer,id in ids.items():
reqBlocks=[b for b in F.keys() if b in requestableBlocks.keys() and F[b].has_key(id) and int(F[b][id])==1]
blocksToRequest[peer] +=reqBlocks
for b in reqBlocks:
del requestableBlocks[b]
keys = tmpBlocksToRequest.keys()
blocksToRequest = {}
for k in keys:
blocksToRequest[k] = []
#take out blocks with only 1 source
reqBlockList = requestableBlocks.keys()
for b in reqBlockList:
if len(requestableBlocks[b]) == 1:
peer = requestableBlocks[b][0]
blocksToRequest[peer].append(b)
del requestableBlocks[b]
"""
#while There are blocks to request
while len(requestableBlocks) > 0:
#get the block with less sources
block = min([(len(requestableBlocks[x]), x)
for x in requestableBlocks])[1]
#get the peer with min(less possible requests, less requests so far)
peer = min([(min(len(tmpBlocksToRequest[x]),
len(blocksToRequest[x])), x)
for x in tmpBlocksToRequest
if block in tmpBlocksToRequest[x]])[1]
del requestableBlocks[block]
blocksToRequest[peer].append(block)
for k, v in blocksToRequest.items():
if len(v) > 3:
blocksToRequest[k] = v[:3]
#self.log.debug('requesting blocks %s',blocksToRequest)
# print "BLOCKS TO REQUEST:"
# print blocksToRequest
return blocksToRequest
return deferToThread(dd, self, receivingBlocks, missingBlocks,
neighbours)
#return dd(self, receivingBlocks, missingBlocks, neighbours)
def sendRequests(self, requests):
for peer in self.overlay.getNeighbours():
r = requests.get(peer)
#self.log.debug('sending requests to %s %s',peer,r)
bw = self.root.trafficPipe.callSimple('getReportedBw')
BufferMessage.send(self.stream.id, bw, self.buffer, r, peer,
self.controlPipe)
def sendLPB(self, peer):
self.log.warning('sending LPB message to %s', peer)
LPBMessage.send(self.stream.id, self.buffer.lpb, peer,
self.controlPipe)
def shift(self, norequests=False):
n = self.overlay.getNeighbours()
outID, hit = self.buffer.shift()
setLPB(self, self.buffer.lpb)
if self.buffer.lpb - self.buffer.flpb > self.buffer.buffersize:
if not hit:
self.countMiss += 1
else:
self.countHit += 1
hitRatio = self.countHit / float(self.countHit + self.countMiss)
setValue(self, 'scheduler', hitRatio * 1000)
#self.log.debug('hit ratio %f',hitRatio)
if not norequests:
#send buffers
if self.buffer.lpb % self.reqInterval == 0:
d = self.trafficPipe.call("getreceiving", self)
d.addCallback(self.makeRequests, self.buffer.getFalseBIDList(),
n)
d.addCallback(self.sendRequests)
d.addErrback(self.errback)
else:
#print 'sending buffer'
bw = self.root.trafficPipe.callSimple('getReportedBw')
BufferMessage.send(self.stream.id, bw, self.buffer, None, n,
self.controlPipe)
#self.log.debug('%s',self.buffer)
try:
idleRatio = self.idleTime / (time() - self.startTime)
except:
idleRatio = 0
#self.log.debug('idle:%f',idleRatio)
setValue(self, 'idle', idleRatio * 1000)
#print self.buffer
#push block to output
outdata = self.trafficPipe.call("popblockdata", self, outID)
outdata.addCallback(self.output.write)
def isRunning(self):
return self.loopingCall.running
def askFragments(self, bid, fragments, peer):
print 'should ask from ', peer, fragments, bid
self.log.warning('should ask from %s,%s,%d', peer, fragments, bid)
RetransmitMessage.send(self.stream.id, fragments, bid, peer,
self.controlPipe)
def retransmit(self, block, fragments, peer):
print 'should retransmit to ', peer, block, fragments
self.log.warning('should retransmit to %s,%d,%s', peer, block,
fragments)
b = {}
b['blockid'] = block
b['fragments'] = fragments
self.trafficPipe.call('sendFragments', self, b, peer)
class SimpleProducer(Scheduler):
def registerMessages(self):
self.messages = []
self.messages.append(RetransmitMessage())
def initScheduler(self):
self.log.info('initing producer scheduler')
self.registerMessages()
self.loopingCall = task.LoopingCall(self.shift)
self.blocksSec = self.stream.scheduler['blocksec']
self.frequency = 1.0/self.stream.scheduler['blocksec']
self.buffer = Buffer(buffersize=self.stream.scheduler['bufsize'],log=self.log)
self.blockCache = {}
def produceBlock(self):
pass
def sendLPB(self, peer):
print peer
self.log.debug('sending LPB message to %s',peer)
ServerLPBMessage.send(self.stream.id, self.buffer.lpb, peer, self.controlPipe)
def start(self):
self.log.info('producer scheduler starts running')
self.loopingCall.start(self.frequency)
def stop(self):
self.log.info('producer scheduler is stopping')
try:
self.loopingCall.stop()
except:
pass
self.buffer = Buffer(buffersize=self.stream.scheduler['bufsize'],log=self.log)
self.blockCache = {}
def shift(self):
from time import time
#print "SHIFT: ", time()
#self.log.debug("SHIFT: %f",time())
chunk = self.input.read()
lpb = self.buffer.lpb
#print self.buffer
if chunk is None:
self.log.warning('empty chunk')
self.stop()
return "EOF"
if len(chunk) > 0:
self.buffer.update(lpb)
d = self.trafficPipe.call("inputblock", self, lpb, chunk)
destination = self.overlay.getNeighbours()
if len(destination)>0:
sent=[]
for i in range(min(len(destination),2)):
gp=[p for p in destination if '150.140.186' in p.getIP() and p not in sent]
if gp:
np=gp
else:
np=[p for p in destination if p not in sent]
if np:
peer=choice(np)
self.log.debug('sending block to %s %d %d',peer,self.buffer.lpb,len(chunk))
d.addCallback(self.sendblock, lpb, peer)
sent.append(peer)
outID,hit = self.buffer.shift()
setLPB(self, self.buffer.lpb)
#self.log.debug('%s',self.buffer)
outdata = self.trafficPipe.call("popblockdata", self, outID)
outdata.addCallback(self.output.write)
counter(self, "sent_block")
setValue(self,'scheduler',1000)
def sendblock(self, r, bid, peer):
return self.trafficPipe.call("sendblock", self, bid, peer)
def isRunning(self):
return self.loopingCall.running
def retransmit(self,block,fragments,peer):
print 'should retransmit to ',peer,block,fragments
b={}
b['blockid']=block
b['fragments']=fragments
self.trafficPipe.call('sendFragments',self,b,peer)
class SPullClient(Scheduler):
def registerMessages(self):
self.messages = []
self.messages.append(BufferMessage())
self.messages.append(LPBMessage())
self.messages.append(RetransmitMessage())
self.blocks = []
self.blocks.append(Block())
def initScheduler(self):
self.log.info('initing scheduler')
self.running = False
self.registerMessages()
self.loopingCall = task.LoopingCall(self.shift)
self.reqInterval = self.stream.scheduler['reqInt']
self.frequency = 1.0 / self.stream.scheduler['blocksec']
self.buffer = Buffer(buffersize=self.stream.scheduler['bufsize'],
log=self.log)
def errback(self, failure):
return failure
def produceBlock(self):
#print "PRODUCEBLOCK"
d = deferToThread(self.getRequestedBID)
d.addCallback(self.sendBlock)
d.addErrback(self.errback)
return d
def sendBlock(self, req):
if not req:
self.running = False
return None
bid, peer = req
#self.log.debug('sending block %d to %s',bid,peer)
self.trafficPipe.call("sendblock", self, bid, peer)
def getRequestedBID(self):
#print "GETREQUESTEDBID"
while True:
#print self.bufferlist
peer = getMostDeprivedReq(self.bufferlist, self.buffer)
if peer is None:
self.running = False
#print "STOP SERVING\n\n"
return None
bl = self.buffer.bIDListCompTrue(peer.s[self.stream.id]["request"])
if len(bl) > 0:
blockID = choice(bl)
peer.s[self.stream.id]["request"].remove(blockID)
peer.s[self.stream.id]["buffer"].update(blockID)
#print "SENDING BLOCK", blockID, peer
peer.s[self.stream.id]["luck"] = blockID
return (blockID, peer)
else:
peer.s[self.stream.id]["request"] = []
def start(self):
self.log.info('scheduler is starting')
self.loopingCall.start(self.frequency)
def stop(self):
self.log.info('scheduler is stopping')
#reactor.callLater(0, self.stream.stop)
try:
self.loopingCall.stop()
except:
pass
def makeRequests(self, receivingBlocks, missingBlocks, neighbours):
#print 'neighbours:',neighbours
#print "COMPUTING REQUESTS"
#print missingBlocks
#exclude receiving
def dd(self, receivingBlocks, missingBlocks, neighbours):
for bid in missingBlocks:
if bid in receivingBlocks:
missingBlocks.remove(bid)
elif bid < self.buffer.flpb:
missingBlocks.remove(bid)
#print 'missing blocks:',missingBlocks
tmpBlocksToRequest = {}
requestableBlocks = {}
for peer in neighbours:
if self.stream.id not in peer.s:
print str(self.stream.id), 'in cotinue 1', peer, peer.s
print neighbours
continue
if "buffer" not in peer.s[self.stream.id]:
print 'in continue 2'
continue
buffer = peer.s[self.stream.id]["buffer"]
#print 'neigh buffer:',buffer
tempReq = buffer.bIDListCompTrue(missingBlocks)
tmpBlocksToRequest[peer] = tempReq
#print 'temp:',tempReq
for b in tempReq:
if b in requestableBlocks:
requestableBlocks[b].append(peer)
else:
requestableBlocks[b] = [peer]
keys = tmpBlocksToRequest.keys()
blocksToRequest = {}
for k in keys:
blocksToRequest[k] = []
#take out blocks with only 1 source
reqBlockList = requestableBlocks.keys()
for b in reqBlockList:
if len(requestableBlocks[b]) == 1:
peer = requestableBlocks[b][0]
blocksToRequest[peer].append(b)
del requestableBlocks[b]
#while There are blocks to request
while len(requestableBlocks) > 0:
#get the block with less sources
block = min([(len(requestableBlocks[x]), x)
for x in requestableBlocks])[1]
#get the peer with min(less possible requests, less requests so far)
peer = min([(min(len(tmpBlocksToRequest[x]),
len(blocksToRequest[x])), x)
for x in tmpBlocksToRequest
if block in tmpBlocksToRequest[x]])[1]
del requestableBlocks[block]
blocksToRequest[peer].append(block)
#print "BLOCKSTOREQUESTSSSS", blocksToRequest
peerWithRequests = 0
for peer in blocksToRequest:
if len(blocksToRequest[peer]):
peerWithRequests += 1
if peerWithRequests > self.reqInterval:
blocksToRequest = requestOne(blocksToRequest)
return blocksToRequest
return deferToThread(dd, self, receivingBlocks, missingBlocks,
neighbours)
#return dd(self, receivingBlocks, missingBlocks, neighbours)
def sendRequests(self, requests):
for peer in self.overlay.getNeighbours():
reactor.callLater(uniform(0, 0.05),
BufferMessage.send, self.stream.id, self.buffer,
requests.get(peer), peer, self.controlPipe)
def sendLPB(self, peer):
self.log.warning('sending LPB message to %s', peer)
LPBMessage.send(self.stream.id, self.buffer.lpb, peer,
self.controlPipe)
def shift(self, norequests=False):
n = self.overlay.getNeighbours()
outID = self.buffer.shift()
if not norequests:
#send buffers
if self.buffer.lpb % self.reqInterval == 0:
d = self.trafficPipe.call("getreceiving", self)
d.addCallback(self.makeRequests, self.buffer.getFalseBIDList(),
n)
d.addCallback(self.sendRequests)
d.addErrback(self.errback)
else:
#print 'sending buffer'
reactor.callLater(uniform(0, 0.05), BufferMessage.send,
self.stream.id, self.buffer, None, n,
self.controlPipe)
#self.log.debug('%s',self.buffer)
#print self.buffer
#push block to output
outdata = self.trafficPipe.call("popblockdata", self, outID)
outdata.addCallback(self.output.write)
def isRunning(self):
return self.loopingCall.running
def askFragments(self, bid, fragments, peer):
print 'should ask from ', peer, fragments, bid
RetransmitMessage.send(self.stream.id, fragments, bid, peer,
self.controlPipe)
def retransmit(self, block, fragments, peer):
print 'should retransmit to ', peer, block, fragments
b = {}
b['blockid'] = block
b['fragments'] = fragments
self.trafficPipe.call('sendFragments', self, b, peer)