def __init__(self, bidder_params, logger): # INIT properties self.bidder_params = bidder_params self.peername = bidder_params['peer'] self.silent = bidder_params['silent'] self.streaming_url = bidder_params['url'] self.buffer_folder = setting.BUFFER_DIR # discovery center self.discovery_center = Broadcaster( bidder_params['broadcast'], setting.DIS_BID_PORT) # message center self.message_center = Message( setting.MSG_HOST, setting.MSG_BID_PORT, setting.MSG_AUC_PORT, BidderProtocol(self)) # transport center self.transport_center = transport.TransportServer( setting.TRP_HOST, setting.TRP_PORT, TransportProtocol(self)) # player self.player = BidderPlayer(self) # log center self.logger = logger #self.logger = log.LogClient(peername, bidder_params['broadcast']) self.running = 0
def __init__(self, bidder_params, logger):
# INIT properties
self.bidder_params = bidder_params
self.peername = bidder_params['peer']
self.silent = bidder_params['silent']
self.streaming_url = bidder_params['url']
self.buffer_folder = setting.BUFFER_DIR
# discovery center
self.discovery_center = Broadcaster(bidder_params['broadcast'],
setting.DIS_BID_PORT)
# message center
self.message_center = Message(setting.MSG_HOST, setting.MSG_BID_PORT,
setting.MSG_AUC_PORT,
BidderProtocol(self))
# transport center
self.transport_center = transport.TransportServer(
setting.TRP_HOST, setting.TRP_PORT, TransportProtocol(self))
# player
self.player = BidderPlayer(self)
# log center
self.logger = logger #self.logger = log.LogClient(peername, bidder_params['broadcast'])
self.running = 0
def __init__(self, peername, logfile):
self.peername = peername
# discovery module
self.bcaster = Broadcaster(setting.UDP_BROADCAST,
setting.CTR_MASTER_BCAST_PORT)
# message module
self.message = Message(setting.CTR_HOST, setting.CTR_MASTER_PORT,
setting.CTR_SLAVE_PORT, self)
# slaves
self.slaves = {}
self.peers = {}
# logging
self.logfile = logfile if logfile else '%s-%s' % (
time.strftime("%m-%d"), str(int(time.time()) % 100).zfill(3))
self.logfile = os.path.join(setting.LOG_DIR, self.logfile)
self.logger = logging.getLogger()
self.logger.setLevel(logging.INFO)
fhandler = logging.FileHandler(self.logfile + '.log')
formatter = logging.Formatter('%(asctime)s - %(message)s')
fhandler.setFormatter(formatter)
self.logger.addHandler(fhandler)
'Plot module'
import plot
self.plot = plot
def __init__(self, peername, logfile):
self.peername = peername
# discovery module
self.bcaster = Broadcaster(setting.UDP_BROADCAST, setting.CTR_MASTER_BCAST_PORT)
# message module
self.message = Message(setting.CTR_HOST, setting.CTR_MASTER_PORT, setting.CTR_SLAVE_PORT, self)
# slaves
self.slaves = {}
self.peers = {}
# logging
self.logfile = logfile if logfile else '%s-%s' % (time.strftime("%m-%d"),str(int(time.time()) % 100).zfill(3))
self.logfile = os.path.join(setting.LOG_DIR, self.logfile)
self.logger = logging.getLogger()
self.logger.setLevel(logging.INFO)
fhandler = logging.FileHandler(self.logfile+'.log')
formatter = logging.Formatter('%(asctime)s - %(message)s')
fhandler.setFormatter(formatter)
self.logger.addHandler(fhandler)
'Plot module'
import plot
self.plot = plot
class Master(MessageProtocol):
def __init__(self, peername, logfile):
self.peername = peername
# discovery module
self.bcaster = Broadcaster(setting.UDP_BROADCAST, setting.CTR_MASTER_BCAST_PORT)
# message module
self.message = Message(setting.CTR_HOST, setting.CTR_MASTER_PORT, setting.CTR_SLAVE_PORT, self)
# slaves
self.slaves = {}
self.peers = {}
# logging
self.logfile = logfile if logfile else '%s-%s' % (time.strftime("%m-%d"),str(int(time.time()) % 100).zfill(3))
self.logfile = os.path.join(setting.LOG_DIR, self.logfile)
self.logger = logging.getLogger()
self.logger.setLevel(logging.INFO)
fhandler = logging.FileHandler(self.logfile+'.log')
formatter = logging.Formatter('%(asctime)s - %(message)s')
fhandler.setFormatter(formatter)
self.logger.addHandler(fhandler)
'Plot module'
import plot
self.plot = plot
#def loginfo(self, message):
# print '%s %s\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), message)
# #self.loghandler.write('%s %s\n', time.strftime('%Y-%m-%d %H:%M:%S'), message)
'Open API'
def run(self):
self.bcaster.run()
self.message.run()
self.plot_rate_init()
def close(self):
self.bcaster.close()
self.message.close()
self.plot_rate_finish()
def send_order_to_slave(self, peername, order):
if peername in self.slaves:
self.message.sendto(self.slaves[peername], order)
def start_bidder_of_slave(self, peername):
self.send_order_to_slave(peername, 'B1')
def stop_bidder_of_slave(self, peername):
self.send_order_to_slave(peername, 'B2')
def start_auctioneer_of_slave(self, peername):
self.send_order_to_slave(peername, 'A1')
def stop_auctioneer_of_slave(self, peername):
self.send_order_to_slave(peername, 'A2')
'MessageProtocol'
def on_msg_from_peer(self, data, peer):
# parse log message
try:
tag, pack = data.split(':',1)
except:
return
# response
if tag == 'INT':#introduce
self.slaves[pack] = peer
self.peers[peer] = pack
self.logger.info('%s@join ip=%s' % (pack, peer))
self.on_slave_join(pack)
else:
try:
peername = self.peers[peer]
except:
return
if tag == 'P':
self.on_slave_play(peername, pack)
elif tag == 'B':
self.on_slave_bid(peername, pack)
elif tag == 'A':
self.on_slave_auction(peername, pack)
elif tag == 'D':
self.on_slave_decide(peername, pack)
elif tag == 'T':
self.on_slave_transport(peername, pack)
'Recall API'
def on_slave_play(self, peername, pack):
rate, duration, delay = struct.unpack('!fff', pack)
self.plot_rate_add((rate,duration,delay))
self.logger.info('%s@play rate=%0.2f, duration=%0.2f, rebuffer=%0.2f'% (peername, rate, duration, delay))
def on_slave_bid(self, peername, pack):
self.logger.info('%s@bid %s' % (peername, pack))
def on_slave_auction(self, peername, pack):
self.logger.info('%s@auction %s' % (peername, pack))
def on_slave_decide(self, peername, pack):
#[self.auction_index-1, ip, allocs[ip][0], allocs[ip][1], allocs[ip][2]]
index, ip, tasks, bitrate, price = eval(pack)
try:
bidderpeer = self.peers[ip]
except:
return
self.logger.info('%s@decide index=%d, winner=%s, bitrate=%0.2f, price=%0.2f, segments=%d' % (peername, index, bidderpeer, bitrate/1024.0/1024.0, price, tasks))
def on_slave_transport(self, peername, pack):
#[ip, index, size, duration]
ip, index, size, duration = eval(pack)
try:
bidderpeer = self.peers[ip]
except:
return
self.logger.info('%s@transport index=%d, bidder=%s, size=%0.2f, transport_duration=%0.2f' %(peername, int(index), bidderpeer, size, duration))
#Overide
def on_slave_join(self, peername):
pass
'Plot API'
def plot_rate_init(self):
self.rate_items = []
def plot_rate_add(self, item):
self.rate_items.append(item)
def plot_rate_finish(self):
self.plot.create_rate_plot(self.rate_items, self.logfile+'_rate.png')
class Bidder(object):
def __init__(self, bidder_params, logger):
# INIT properties
self.bidder_params = bidder_params
self.peername = bidder_params['peer']
self.silent = bidder_params['silent']
self.streaming_url = bidder_params['url']
self.buffer_folder = setting.BUFFER_DIR
# discovery center
self.discovery_center = Broadcaster(
bidder_params['broadcast'],
setting.DIS_BID_PORT)
# message center
self.message_center = Message(
setting.MSG_HOST,
setting.MSG_BID_PORT,
setting.MSG_AUC_PORT,
BidderProtocol(self))
# transport center
self.transport_center = transport.TransportServer(
setting.TRP_HOST,
setting.TRP_PORT,
TransportProtocol(self))
# player
self.player = BidderPlayer(self)
# log center
self.logger = logger #self.logger = log.LogClient(peername, bidder_params['broadcast'])
self.running = 0
""" Bidder(receiver, player) Life Cycle """
def run(self):
print ''
print '### Bidder', self.peername, '( kcapacity = ', self.bidder_params['kcapacity'],') running...'
print ''
self.running = 1
self.prepare()
self.player.play()
self.discovery_center.run()
self.message_center.run()
self.transport_center.start()
def join(self):
self.transport_center.join()
def close(self):
self.discovery_center.close()
self.message_center.close()
self.transport_center.close()
self.player.close()
self.running = 0
self.task_timeout_cond.acquire()
self.task_timeout_cond.notify()
self.task_timeout_cond.release()
print ''
print '### Bidder', self.peername, 'stopped'
print ''
def prepare(self):
self.player.prepare2play()
# the index of which trunk is ready to write into buffer
self.ready_index = 0
# wait for auction : priority queue [segment number]
self.auction_waiting_queue = PriorityQueue()
for i in range(self.player.get_segment_number()):
self.auction_waiting_queue.put(i)
# wait for retrieve : dictionary { index: (rate,time) }
self.retrieving = {}
# retrieved : dictionary { index:(rate,data) }
self.retrieved = {}
self.task_cond = threading.Condition()
threading.Thread(target=self.task_loop, args=(1.0, )).start()
# timeout protection
self.task_timeout_cond = threading.Condition()
threading.Thread(target=self.task_timeout, args=(setting.AUCTIONEER_DOWNLOAD_TIMEOUT,)).start()
# core
self.core = BidderCore(self, self.bidder_params)
def buffer_size(self):
buffer_in_player = self.player.get_buffer()
buffer_in_auction = self.player.get_segment_duration() * (len(self.retrieved) + len(self.retrieving))
return buffer_in_player + buffer_in_auction
""" Bid Factory.
bid : bid for auction.
send task : send segment task for auction.
"""
def bid(self, ip, auction):
bid_pack = self.core.bid2auction(auction)
if bid_pack:
# unpack
auction_peer, auction_index, bid = bid_pack
# logging
#self.logger.log('B', [self.peername, auction_peer, auction_index, self.buffer_size(), bid])
logcontent = 'auctioneer=%s, index=%d, buffer=%0.2f, bitrates=%s, prices=%s, gains=%s' % (auction_peer, int(auction_index), self.buffer_size(), str([ r/1024.0/1024.0 for r in bid[0]]), str(bid[1]), str(bid[2]))
self.logger.slave_bid(logcontent)
# response
bid_info = ','.join([auction_index, str(bid)])
self.message_center.sendto(ip, ':'.join(['BID', bid_info]))
def send_task(self, ip, info):
segment_allocated, rate = map(lambda a:int(a), info.split(','))
self.core.update_previous(rate)
while segment_allocated > 0 and not self.auction_waiting_queue.empty():
index = self.auction_waiting_queue.get()
task_url = self.player.get_segment_url(index, rate)
f_rate = float(rate)/1024/1024
print '[B sended] No.%d, rate=%0.2f(mbps)' % (index, f_rate)
# send it to bidder
self.retrieving[index] = (f_rate, time.time())
self.message_center.sendto(ip, 'TASK:'+str(index)+','+task_url)
segment_allocated = segment_allocated - 1
""" Transport Factory """
def receive_trunk(self, ip, index, data):
print '[B received] No.%s, size=%0.2f(mb), buffer=%0.2f(s)' % (index, float(len(data))/1024/128, self.buffer_size())
# discard wild data
if not index in self.retrieving:
return
# retrieve thread safe
self.task_cond.acquire()
self.retrieved[index] = (self.retrieving[index][0], data)
del self.retrieving[index]
self.task_cond.notify()
self.task_cond.release()
def fail_trunk(self, ip, index):
if not index in self.retrieving: #wild data
return
del self.retrieving[index]
self.auction_waiting_queue.put(index)
""" Task """
def task_timeout(self, timeout):
while self.running:
now = time.time()
for index in self.retrieving:#TODO thread safe
if self.retrieving[index][1] - now > timeout:
del self.retrieving[index]
self.auction_waiting_queue.put(index)
#time.sleep(timeout)
self.task_timeout_cond.acquire()
self.task_timeout_cond.wait(timeout)
self.task_timeout_cond.release()
def task_loop(self, timeout):
while self.running:
self.task_cond.acquire()
while self.running and not self.ready_index in self.retrieved:
self.task_cond.wait(timeout)
if not self.running:
break
if self.ready_index in self.retrieved:
self.player.segment_received(self.ready_index, self.retrieved[self.ready_index])
del self.retrieved[self.ready_index]
self.ready_index = self.ready_index + 1
self.task_cond.release()
class Bidder(object):
def __init__(self, bidder_params, logger):
# INIT properties
self.bidder_params = bidder_params
self.peername = bidder_params['peer']
self.silent = bidder_params['silent']
self.streaming_url = bidder_params['url']
self.buffer_folder = setting.BUFFER_DIR
# discovery center
self.discovery_center = Broadcaster(bidder_params['broadcast'],
setting.DIS_BID_PORT)
# message center
self.message_center = Message(setting.MSG_HOST, setting.MSG_BID_PORT,
setting.MSG_AUC_PORT,
BidderProtocol(self))
# transport center
self.transport_center = transport.TransportServer(
setting.TRP_HOST, setting.TRP_PORT, TransportProtocol(self))
# player
self.player = BidderPlayer(self)
# log center
self.logger = logger #self.logger = log.LogClient(peername, bidder_params['broadcast'])
self.running = 0
""" Bidder(receiver, player) Life Cycle """
def run(self):
print ''
print '### Bidder', self.peername, '( kcapacity = ', self.bidder_params[
'kcapacity'], ') running...'
print ''
self.running = 1
self.prepare()
self.player.play()
self.discovery_center.run()
self.message_center.run()
self.transport_center.start()
def join(self):
self.transport_center.join()
def close(self):
self.discovery_center.close()
self.message_center.close()
self.transport_center.close()
self.player.close()
self.running = 0
self.task_timeout_cond.acquire()
self.task_timeout_cond.notify()
self.task_timeout_cond.release()
print ''
print '### Bidder', self.peername, 'stopped'
print ''
def prepare(self):
self.player.prepare2play()
# the index of which trunk is ready to write into buffer
self.ready_index = 0
# wait for auction : priority queue [segment number]
self.auction_waiting_queue = PriorityQueue()
for i in range(self.player.get_segment_number()):
self.auction_waiting_queue.put(i)
# wait for retrieve : dictionary { index: (rate,time) }
self.retrieving = {}
# retrieved : dictionary { index:(rate,data) }
self.retrieved = {}
self.task_cond = threading.Condition()
threading.Thread(target=self.task_loop, args=(1.0, )).start()
# timeout protection
self.task_timeout_cond = threading.Condition()
threading.Thread(target=self.task_timeout,
args=(setting.AUCTIONEER_DOWNLOAD_TIMEOUT, )).start()
# core
self.core = BidderCore(self, self.bidder_params)
def buffer_size(self):
buffer_in_player = self.player.get_buffer()
buffer_in_auction = self.player.get_segment_duration() * (
len(self.retrieved) + len(self.retrieving))
return buffer_in_player + buffer_in_auction
""" Bid Factory.
bid : bid for auction.
send task : send segment task for auction.
"""
def bid(self, ip, auction):
bid_pack = self.core.bid2auction(auction)
if bid_pack:
# unpack
auction_peer, auction_index, bid = bid_pack
# logging
#self.logger.log('B', [self.peername, auction_peer, auction_index, self.buffer_size(), bid])
logcontent = 'auctioneer=%s, index=%d, buffer=%0.2f, bitrates=%s, prices=%s, gains=%s' % (
auction_peer, int(auction_index), self.buffer_size(),
str([r / 1024.0 / 1024.0
for r in bid[0]]), str(bid[1]), str(bid[2]))
self.logger.slave_bid(logcontent)
# response
bid_info = ','.join([auction_index, str(bid)])
self.message_center.sendto(ip, ':'.join(['BID', bid_info]))
def send_task(self, ip, info):
segment_allocated, rate = map(lambda a: int(a), info.split(','))
self.core.update_previous(rate)
while segment_allocated > 0 and not self.auction_waiting_queue.empty():
index = self.auction_waiting_queue.get()
task_url = self.player.get_segment_url(index, rate)
f_rate = float(rate) / 1024 / 1024
print '[B sended] No.%d, rate=%0.2f(mbps)' % (index, f_rate)
# send it to bidder
self.retrieving[index] = (f_rate, time.time())
self.message_center.sendto(ip,
'TASK:' + str(index) + ',' + task_url)
segment_allocated = segment_allocated - 1
""" Transport Factory """
def receive_trunk(self, ip, index, data):
print '[B received] No.%s, size=%0.2f(mb), buffer=%0.2f(s)' % (
index, float(len(data)) / 1024 / 128, self.buffer_size())
# discard wild data
if not index in self.retrieving:
return
# retrieve thread safe
self.task_cond.acquire()
self.retrieved[index] = (self.retrieving[index][0], data)
del self.retrieving[index]
self.task_cond.notify()
self.task_cond.release()
def fail_trunk(self, ip, index):
if not index in self.retrieving: #wild data
return
del self.retrieving[index]
self.auction_waiting_queue.put(index)
""" Task """
def task_timeout(self, timeout):
while self.running:
now = time.time()
for index in self.retrieving: #TODO thread safe
if self.retrieving[index][1] - now > timeout:
del self.retrieving[index]
self.auction_waiting_queue.put(index)
#time.sleep(timeout)
self.task_timeout_cond.acquire()
self.task_timeout_cond.wait(timeout)
self.task_timeout_cond.release()
def task_loop(self, timeout):
while self.running:
self.task_cond.acquire()
while self.running and not self.ready_index in self.retrieved:
self.task_cond.wait(timeout)
if not self.running:
break
if self.ready_index in self.retrieved:
self.player.segment_received(self.ready_index,
self.retrieved[self.ready_index])
del self.retrieved[self.ready_index]
self.ready_index = self.ready_index + 1
self.task_cond.release()
class Master(MessageProtocol):
def __init__(self, peername, logfile):
self.peername = peername
# discovery module
self.bcaster = Broadcaster(setting.UDP_BROADCAST,
setting.CTR_MASTER_BCAST_PORT)
# message module
self.message = Message(setting.CTR_HOST, setting.CTR_MASTER_PORT,
setting.CTR_SLAVE_PORT, self)
# slaves
self.slaves = {}
self.peers = {}
# logging
self.logfile = logfile if logfile else '%s-%s' % (
time.strftime("%m-%d"), str(int(time.time()) % 100).zfill(3))
self.logfile = os.path.join(setting.LOG_DIR, self.logfile)
self.logger = logging.getLogger()
self.logger.setLevel(logging.INFO)
fhandler = logging.FileHandler(self.logfile + '.log')
formatter = logging.Formatter('%(asctime)s - %(message)s')
fhandler.setFormatter(formatter)
self.logger.addHandler(fhandler)
'Plot module'
import plot
self.plot = plot
#def loginfo(self, message):
# print '%s %s\n' % (time.strftime('%Y-%m-%d %H:%M:%S'), message)
# #self.loghandler.write('%s %s\n', time.strftime('%Y-%m-%d %H:%M:%S'), message)
'Open API'
def run(self):
self.bcaster.run()
self.message.run()
self.plot_rate_init()
def close(self):
self.bcaster.close()
self.message.close()
self.plot_rate_finish()
def send_order_to_slave(self, peername, order):
if peername in self.slaves:
self.message.sendto(self.slaves[peername], order)
def start_bidder_of_slave(self, peername):
self.send_order_to_slave(peername, 'B1')
def stop_bidder_of_slave(self, peername):
self.send_order_to_slave(peername, 'B2')
def start_auctioneer_of_slave(self, peername):
self.send_order_to_slave(peername, 'A1')
def stop_auctioneer_of_slave(self, peername):
self.send_order_to_slave(peername, 'A2')
'MessageProtocol'
def on_msg_from_peer(self, data, peer):
# parse log message
try:
tag, pack = data.split(':', 1)
except:
return
# response
if tag == 'INT': #introduce
self.slaves[pack] = peer
self.peers[peer] = pack
self.logger.info('%s@join ip=%s' % (pack, peer))
self.on_slave_join(pack)
else:
try:
peername = self.peers[peer]
except:
return
if tag == 'P':
self.on_slave_play(peername, pack)
elif tag == 'B':
self.on_slave_bid(peername, pack)
elif tag == 'A':
self.on_slave_auction(peername, pack)
elif tag == 'D':
self.on_slave_decide(peername, pack)
elif tag == 'T':
self.on_slave_transport(peername, pack)
'Recall API'
def on_slave_play(self, peername, pack):
rate, duration, delay = struct.unpack('!fff', pack)
self.plot_rate_add((rate, duration, delay))
self.logger.info('%s@play rate=%0.2f, duration=%0.2f, rebuffer=%0.2f' %
(peername, rate, duration, delay))
def on_slave_bid(self, peername, pack):
self.logger.info('%s@bid %s' % (peername, pack))
def on_slave_auction(self, peername, pack):
self.logger.info('%s@auction %s' % (peername, pack))
def on_slave_decide(self, peername, pack):
#[self.auction_index-1, ip, allocs[ip][0], allocs[ip][1], allocs[ip][2]]
index, ip, tasks, bitrate, price = eval(pack)
try:
bidderpeer = self.peers[ip]
except:
return
self.logger.info(
'%s@decide index=%d, winner=%s, bitrate=%0.2f, price=%0.2f, segments=%d'
% (peername, index, bidderpeer, bitrate / 1024.0 / 1024.0, price,
tasks))
def on_slave_transport(self, peername, pack):
#[ip, index, size, duration]
ip, index, size, duration = eval(pack)
try:
bidderpeer = self.peers[ip]
except:
return
self.logger.info(
'%s@transport index=%d, bidder=%s, size=%0.2f, transport_duration=%0.2f'
% (peername, int(index), bidderpeer, size, duration))
#Overide
def on_slave_join(self, peername):
pass
'Plot API'
def plot_rate_init(self):
self.rate_items = []
def plot_rate_add(self, item):
self.rate_items.append(item)
def plot_rate_finish(self):
self.plot.create_rate_plot(self.rate_items, self.logfile + '_rate.png')