def __init__(self, peername, auctioneer_params):
self.open_servers = {}
# propertys
self.peername = peername
self.delay = auctioneer_params['delay'] if auctioneer_params['delay'] > 1.0 else 1.0
self.auctioneer_params = auctioneer_params
# auction message server
self.message_server = message.MessageServer(
setting.UDP_HOST,
setting.UDP_AUCTION_PORT,
AuctionProtocol(self))
# auction sender
self.message_client = message.MessageClient(
self.auctioneer_params['broadcast'],
setting.UDP_BID_PORT,
message.Protocol())
# transport center
self.transport = transport.TransportClient(
setting.TRP_PORT,
TransportProtocol(self))
# log center
self.logger = log.LogClient(peername, self.auctioneer_params['broadcast'])
self.logger.add_peer(peername)
# algorithm core
self.core = AuctioneerCore(self, self.auctioneer_params)
def __init__(self, auctioneer_params, logger): # propertys self.auctioneer_params = auctioneer_params self.peername = auctioneer_params['peer'] self.delay = auctioneer_params['delay'] if auctioneer_params['delay'] > 1.0 else 1.0 # discovery center self.discovery_center = Listener( setting.DIS_HOST,#auctioneer_params['broadcast'], setting.DIS_BID_PORT, self) # message center self.message_center = Message( setting.MSG_HOST, setting.MSG_AUC_PORT, setting.MSG_BID_PORT, AuctionProtocol(self)) # transport center self.transport = transport.TransportClient( setting.TRP_PORT, transport.Protocol()) # log center self.logger = logger #self.logger = log.LogClient(peername, self.auctioneer_params['broadcast']) # algorithm core self.core = AuctioneerCore(self, self.auctioneer_params) self.running = 0
def __init__(self, auctioneer_params, logger):
# propertys
self.auctioneer_params = auctioneer_params
self.peername = auctioneer_params['peer']
self.delay = auctioneer_params[
'delay'] if auctioneer_params['delay'] > 1.0 else 1.0
# discovery center
self.discovery_center = Listener(
setting.DIS_HOST, #auctioneer_params['broadcast'],
setting.DIS_BID_PORT,
self)
# message center
self.message_center = Message(setting.MSG_HOST, setting.MSG_AUC_PORT,
setting.MSG_BID_PORT,
AuctionProtocol(self))
# transport center
self.transport = transport.TransportClient(setting.TRP_PORT,
transport.Protocol())
# log center
self.logger = logger #self.logger = log.LogClient(peername, self.auctioneer_params['broadcast'])
# algorithm core
self.core = AuctioneerCore(self, self.auctioneer_params)
self.running = 0
class Auctioneer(ListenerProtocol):
def __init__(self, auctioneer_params, logger):
# propertys
self.auctioneer_params = auctioneer_params
self.peername = auctioneer_params['peer']
self.delay = auctioneer_params[
'delay'] if auctioneer_params['delay'] > 1.0 else 1.0
# discovery center
self.discovery_center = Listener(
setting.DIS_HOST, #auctioneer_params['broadcast'],
setting.DIS_BID_PORT,
self)
# message center
self.message_center = Message(setting.MSG_HOST, setting.MSG_AUC_PORT,
setting.MSG_BID_PORT,
AuctionProtocol(self))
# transport center
self.transport = transport.TransportClient(setting.TRP_PORT,
transport.Protocol())
# log center
self.logger = logger #self.logger = log.LogClient(peername, self.auctioneer_params['broadcast'])
# algorithm core
self.core = AuctioneerCore(self, self.auctioneer_params)
self.running = 0
""" Auctioneer Life Cycle"""
def run(self):
print ''
print '### Auctioneer', self.peername, '( delay = ', self.auctioneer_params[
'delay'], ') running...'
print ''
# INIT
self.running = 1
self.transport_queue = Queue()
self.bids = {} # bids {ip:bid}
self.tasks = {} # tasks {ip:task number}
self.auction_index = 0
# launch
self.discovery_center.run()
self.message_center.run()
threading.Thread(target=self.auction_loop).start() # join ignore
def close(self):
self.discovery_center.close()
self.message_center.close()
self.running = 0
print ''
print '### Auctioneer', self.peername, 'stopped.'
print ''
def auction_loop(self):
# timeout mechanism
while self.running:
try:
ip, task = self.transport_queue.get(timeout=0.3)
except:
#Time out
self.auction()
time.sleep(0.1)
#repeated broadcast
''' deprecated
for i in range(3):
self.auction()
time.sleep(0.033)
'''
self.decide_auction()
else:
if not ip in self.tasks or self.tasks[ip] <= 0:
continue
index, url = task.split(',', 1)
size, duration, success = self.transport.transport(
ip, index, url)
size = float(size) / 1024 / 128 #bytes to mb
#delay
if self.delay > 1.0:
#print 'delay', duration * (self.delay - 1.0)
time.sleep(duration * (self.delay - 1.0))
duration = duration * self.delay
capacity = size / duration if duration > 0 else self.auctioneer_params[
'capacity']
self.core.estimate_capacity(capacity)
self.tasks[ip] = self.tasks[ip] - 1
if success:
#logging
#self.logger.log('T', [ip, index, size, duration])#self.logger.transport_complete(ip, index, size, duration)
self.logger.slave_transport([ip, index, size, duration])
print '[A completed] No.%s, size=%0.2f(mb), capacity=%0.2f~%0.2f(mbps), url=%s, at %s' % (
index, size, capacity, self.core.capacity, url,
time.strftime("%H:%M:%S"))
else:
print '[A failed] No.%s' % index
""" Auction Factory.
auction : make an auction.
receive_bid : receive a bid.
"""
def auction(self):
# logging
'''
self.logger.log('A', [
self.peername,
self.auction_index,
self.auctioneer_params['segment'],
self.core.capacity,
self.auctioneer_params['timecost'],
self.auctioneer_params['cellular'],
self.auctioneer_params['wifi']
])'''
logcontent = 'index=%d, segments=%d, capacity=%0.2f, ' % (
self.auction_index, self.auctioneer_params['segment'],
self.core.capacity)
self.logger.slave_auction(logcontent)
# broadcast
self.bids.clear()
auction_info = self.core.auction_message(self.auction_index)
for peer in self.discovery_center.peers.keys():
self.message_center.sendto(peer,
':'.join(['AUCTION', auction_info]))
def receive_bid(self, ip, bid):
index, bid_details = bid.split(',', 1)
if int(index) == self.auction_index:
self.bids[ip] = bid_details
def decide_auction(self):
if not self.bids: #receive no bids
return
# finish one auction
self.auction_index += 1 # TODO thread safe ,dict size change when iteration
self.tasks.clear()
# dict {ip : (segments, rate, payment)}
allocs = self.core.select_bid(self.bids)
# notify the winner
for ip in allocs:
self.tasks[ip] = allocs[ip][0]
alloc_result = ','.join([str(allocs[ip][0]), str(allocs[ip][1])])
self.message_center.sendto(ip, ':'.join(['WIN', alloc_result]))
# logging
#self.logger.log('D', [self.peername, self.auction_index-1, ip, allocs[ip][0], allocs[ip][1], allocs[ip][2]])
self.logger.slave_decide([
self.auction_index - 1, ip, allocs[ip][0], allocs[ip][1],
allocs[ip][2]
])
# logging
#self.logger.log('C', self.peername)#self.logger.decide_complete(self.peername)
def receive_task(self, ip, task):
self.transport_queue.put((ip, task))
class Auctioneer(object):
def __init__(self, peername, auctioneer_params):
self.open_servers = {}
# propertys
self.peername = peername
self.delay = auctioneer_params['delay'] if auctioneer_params['delay'] > 1.0 else 1.0
self.auctioneer_params = auctioneer_params
# auction message server
self.message_server = message.MessageServer(
setting.UDP_HOST,
setting.UDP_AUCTION_PORT,
AuctionProtocol(self))
# auction sender
self.message_client = message.MessageClient(
self.auctioneer_params['broadcast'],
setting.UDP_BID_PORT,
message.Protocol())
# transport center
self.transport = transport.TransportClient(
setting.TRP_PORT,
TransportProtocol(self))
# log center
self.logger = log.LogClient(peername, self.auctioneer_params['broadcast'])
self.logger.add_peer(peername)
# algorithm core
self.core = AuctioneerCore(self, self.auctioneer_params)
""" Auctioneer Life Cycle"""
def start(self):
print self.peername, 'running...'
# INIT
self.running = 1
self.transport_queue = Queue()
self.bids = {}# bids {ip:bid}
self.tasks = {}# tasks {ip:task number}
self.auction_index = 0
# launch
self.message_server.start()
self.message_client.start()
threading.Thread(target=self.auction_loop).start() # join ignore
def join(self):
self.message_server.join()
self.message_client.join()
def close(self):
self.message_server.close()
self.message_client.close()
self.running = 0
def auction_loop(self):
# timeout mechanism
while self.running:
try:
ip,task = self.transport_queue.get(timeout=0.3)
except:
#Time out
self.auction()
time.sleep(0.1)
self.decide_auction()
else:
if not ip in self.tasks or self.tasks[ip] <= 0:
continue
index, url = task.split(',',1)
remoteAddress = url.split("/")[2]
remoteHost = remoteAddress.split(":")[0]
remotePort = 80 if len(remoteAddress.split(":")) == 1 else int(remoteAddress.split(":")[1])
proxyPort = get_open_port()
p = subprocess.Popen(setting.HTTP_PROXY_COMMAND.split(" ") + [remoteHost, str(remotePort), str(proxyPort)],stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
self.open_servers[proxyPort] = p
urlOverProxy = "http://" + socket.gethostbyname(socket.gethostname()) + ":" + str(proxyPort) + "/" + "/".join(url.split("/")[3:])
#size, duration = self.transport.transport(ip, index, url)
size, duration = self.transport.transport_text(ip, urlOverProxy)
size = float(size) / 1024 / 128 #bytes to mb
#delay
if self.delay > 1.0:
#print 'delay', duration * (self.delay - 1.0)
time.sleep(duration * (self.delay - 1.0))
duration = duration * self.delay
capacity = size / duration if duration > 0 else self.auctioneer_params['capacity']
self.core.estimate_capacity(capacity)
self.tasks[ip] = self.tasks[ip] - 1
#logging
self.logger.transport_complete(ip, index, size, duration)
print '[task completed]No.', index,
print ', size =', round(size,3), '(mb), capacity =', round(capacity,3), '(mbps), url = ', url,
print ', at',time.strftime("%H:%M:%S")
""" Auction Factory.
auction : make an auction.
receive_bid : receive a bid.
"""
def auction(self):
# logging
self.logger.auction_broadcast(self.peername,
self.auction_index,
self.auctioneer_params['segment'],
self.core.capacity,
self.auctioneer_params['timecost'],
self.auctioneer_params['cellular'],
self.auctioneer_params['wifi'])
# broadcast
self.bids.clear()
auction_info = self.core.auction_message(self.auction_index)
self.message_client.broadcast(':'.join(['AUCTION', auction_info]))
def receive_bid(self, ip, bid):
index, bid_details = bid.split(',',1)
if int(index) == self.auction_index:
self.bids[ip] = bid_details
def decide_auction(self):
if not self.bids:#receive no bids
return
# finish one auction
self.auction_index += 1 # TODO thread safe ,dict size change when iteration
self.tasks.clear()
# dict {ip : (segments, rate, payment)}
allocs = self.core.select_bid(self.bids)
# notify the winner
for ip in allocs:
self.tasks[ip] = allocs[ip][0]
alloc_result = ','.join([str(allocs[ip][0]), str(allocs[ip][1])])
self.message_client.sendto(ip, ':'.join(['WIN', alloc_result]))
# logging
self.logger.auction_decide(self.peername, self.auction_index, ip, allocs[ip][0], allocs[ip][1], allocs[ip][2])
# logging
self.logger.decide_complete(self.peername)
def receive_task(self, ip, task):
self.transport_queue.put((ip,task))
class Auctioneer(ListenerProtocol):
def __init__(self, auctioneer_params, logger):
# propertys
self.auctioneer_params = auctioneer_params
self.peername = auctioneer_params['peer']
self.delay = auctioneer_params['delay'] if auctioneer_params['delay'] > 1.0 else 1.0
# discovery center
self.discovery_center = Listener(
setting.DIS_HOST,#auctioneer_params['broadcast'],
setting.DIS_BID_PORT,
self)
# message center
self.message_center = Message(
setting.MSG_HOST,
setting.MSG_AUC_PORT,
setting.MSG_BID_PORT,
AuctionProtocol(self))
# transport center
self.transport = transport.TransportClient(
setting.TRP_PORT,
transport.Protocol())
# log center
self.logger = logger #self.logger = log.LogClient(peername, self.auctioneer_params['broadcast'])
# algorithm core
self.core = AuctioneerCore(self, self.auctioneer_params)
self.running = 0
""" Auctioneer Life Cycle"""
def run(self):
print ''
print '### Auctioneer', self.peername, '( delay = ', self.auctioneer_params['delay'],') running...'
print ''
# INIT
self.running = 1
self.transport_queue = Queue()
self.bids = {}# bids {ip:bid}
self.tasks = {}# tasks {ip:task number}
self.auction_index = 0
# launch
self.discovery_center.run()
self.message_center.run()
threading.Thread(target=self.auction_loop).start() # join ignore
def close(self):
self.discovery_center.close()
self.message_center.close()
self.running = 0
print ''
print '### Auctioneer', self.peername, 'stopped.'
print ''
def auction_loop(self):
# timeout mechanism
while self.running:
try:
ip,task = self.transport_queue.get(timeout=0.3)
except:
#Time out
self.auction()
time.sleep(0.1)
#repeated broadcast
''' deprecated
for i in range(3):
self.auction()
time.sleep(0.033)
'''
self.decide_auction()
else:
if not ip in self.tasks or self.tasks[ip] <= 0:
continue
index, url = task.split(',',1)
size, duration, success = self.transport.transport(ip, index, url)
size = float(size) / 1024 / 128 #bytes to mb
#delay
if self.delay > 1.0:
#print 'delay', duration * (self.delay - 1.0)
time.sleep(duration * (self.delay - 1.0))
duration = duration * self.delay
capacity = size / duration if duration > 0 else self.auctioneer_params['capacity']
self.core.estimate_capacity(capacity)
self.tasks[ip] = self.tasks[ip] - 1
if success:
#logging
#self.logger.log('T', [ip, index, size, duration])#self.logger.transport_complete(ip, index, size, duration)
self.logger.slave_transport([ip, index, size, duration])
print '[A completed] No.%s, size=%0.2f(mb), capacity=%0.2f~%0.2f(mbps), url=%s, at %s' % (index, size, capacity, self.core.capacity, url, time.strftime("%H:%M:%S"))
else:
print '[A failed] No.%s' % index
""" Auction Factory.
auction : make an auction.
receive_bid : receive a bid.
"""
def auction(self):
# logging
'''
self.logger.log('A', [
self.peername,
self.auction_index,
self.auctioneer_params['segment'],
self.core.capacity,
self.auctioneer_params['timecost'],
self.auctioneer_params['cellular'],
self.auctioneer_params['wifi']
])'''
logcontent = 'index=%d, segments=%d, capacity=%0.2f, ' % (self.auction_index,self.auctioneer_params['segment'],self.core.capacity)
self.logger.slave_auction(logcontent)
# broadcast
self.bids.clear()
auction_info = self.core.auction_message(self.auction_index)
for peer in self.discovery_center.peers.keys():
self.message_center.sendto(peer, ':'.join(['AUCTION', auction_info]))
def receive_bid(self, ip, bid):
index, bid_details = bid.split(',',1)
if int(index) == self.auction_index:
self.bids[ip] = bid_details
def decide_auction(self):
if not self.bids:#receive no bids
return
# finish one auction
self.auction_index += 1 # TODO thread safe ,dict size change when iteration
self.tasks.clear()
# dict {ip : (segments, rate, payment)}
allocs = self.core.select_bid(self.bids)
# notify the winner
for ip in allocs:
self.tasks[ip] = allocs[ip][0]
alloc_result = ','.join([str(allocs[ip][0]), str(allocs[ip][1])])
self.message_center.sendto(ip, ':'.join(['WIN', alloc_result]))
# logging
#self.logger.log('D', [self.peername, self.auction_index-1, ip, allocs[ip][0], allocs[ip][1], allocs[ip][2]])
self.logger.slave_decide([self.auction_index-1, ip, allocs[ip][0], allocs[ip][1], allocs[ip][2]])
# logging
#self.logger.log('C', self.peername)#self.logger.decide_complete(self.peername)
def receive_task(self, ip, task):
self.transport_queue.put((ip,task))