def __Connect2Client(self, number):
self.__connectionLock.acquire()
self.__neighborLock.acquire()
Neighborhood = lminus(self.__ClientNeighborhood,
self.__ClientConnectionIPs)
potentialconnect = sample(Neighborhood, min(len(Neighborhood), number))
for eachClient in potentialconnect:
if self.__NumofConnections < self.__MaxNumConn:
if eachClient not in self.__ClientConnectionIPs:
cliSock = Mytcpsocket()
ack1 = cliSock.Connect2Server(eachClient)
ack2 = cliSock.sendmsg(self.__Addr[1])
ack3 = cliSock.recvmsg(BUFSIZ) # receive ACK
if ack1 and ack2 and ack3 != 'closed':
self.__NumofConnections += 1
self.__ClientConnectionSocks.append(cliSock)
self.__ClientConnectionIPs.append(eachClient)
t = MyThread(self.__upload, (cliSock, eachClient),
self.__upload.__name__)
self.__UploadThreadsLock.acquire()
self.__UploadThreads.append(t)
self.__UploadThreadsLock.release()
self.__BWControlBoot.set()
# start BW control
self.__RateAllocBoot.set()
# start rate allocation
t.start()
sleep(PROCESS_TIME)
else:
self.__ClientNeighborhood.remove(eachClient)
else:
break
self.__connectionLock.release()
self.__neighborLock.release()
def __Connect2Server(self, *targs):
args = targs[0]
number = args[0]
videohash = args[1]
self.__connectionLock.acquire()
self.__neighborLock.acquire()
Neighborhood = lminus(self.__ServerNeighborhood,
self.__ServerConnectionIPs)
potentialconnect = sample(Neighborhood, min(len(Neighborhood), number))
for eachServer in potentialconnect:
if self.__NumofConnections < self.__MaxNumConn:
if eachServer not in self.__ServerConnectionIPs:
servSock = Mytcpsocket()
ack1 = servSock.Connect2Server(eachServer)
ack2 = servSock.sendmsg(self.__Addr[1])
ack3 = servSock.recvmsg(BUFSIZ) # receive ACK
if ack1 and ack2 and ack3 != 'closed':
self.__NumofConnections += 1
self.__ServerConnectionSocks.append(servSock)
self.__ServerConnectionIPs.append(eachServer)
t = MyThread(self.__download,
(servSock, eachServer, videohash),
self.__download.__name__)
t.start()
sleep(PROCESS_TIME)
self.__downloadthreads.append(t)
else:
self.__ServerNeighborhood.remove(eachServer)
else:
break
self.__connectionLock.release()
self.__neighborLock.release()
class MediaServer(object):
'# Author: Hao Zhang\
# Created: 02/11/2011\
# Class: MediaServer - a TCP server that uploads data to users'
def __init__(self, cwd, IS_SERVER=1):
self.__cwd = cwd # current working directory
self.__isserver = IS_SERVER # I mean the True server, not helper
self.__tcpSerSock = Mytcpsocket() # initiate server socket
self.__tcpTracSock = Mytcpsocket() # initiate tracker socket
self.__PATH = self.__cwd + os.sep + SERVER_PATH + os.sep # set up video caching directory
if not os.path.exists(self.__PATH):
os.makedirs(self.__PATH)
# movieList: vhash: vname; videoDict: vhash: vpath; videoStat: vhash: [NumofChunks, NumofLeftOverPieces]
self.__movieList = {}
self.__videoDict = {}
self.__videoStat = {}
# storeAlloc: vhash: vsize; storePrices: vhash: vprice
self.__storeAlloc = {}
self.__storePrices = {}
# mediaReaders: vhash: [a list of open file readers for vhash]; mediaUpdaters: vhash: file writer
self.__mediaReaders = {}
self.__mediaUpdaters = {}
if IS_SERVER:
self.__MaxNumConn = MAX_SERV_CONNECTION
self.__Addr = MEDIA_SERVER_ADDRESS
# read all the movies and convert newly added movies
f = open(self.__PATH + 'movielist.txt', 'r+')
allLines = f.readlines()
f.close()
for eachmovie in allLines:
moviename = eachmovie.split('\n')[0]
self.__movieList[hashlib.md5(
moviename).hexdigest()] = moviename #vhash: vname
dirList = os.listdir(self.__PATH)
for dname in dirList:
if (
not os.path.isdir(self.__PATH + dname)
) and dname != 'movielist.txt': # if video not converted, convert
hashname = hashlib.md5(dname.split('.')[0]).hexdigest()
if (hashname in self.__movieList
and not hashname in dirList):
videopath = self.__PATH + hashname
os.makedirs(videopath)
self.__convertVideo(videopath + os.sep, self.__PATH +
dname) # convert the video
dirList = os.listdir(self.__PATH)
for dname in dirList:
if os.path.isdir(self.__PATH + dname):
videopath = self.__PATH + dname + os.sep
if dname in self.__movieList:
self.__videoDict[dname] = videopath # vhash: vpath
f = open(videopath + 'length', 'rb')
dt = f.readlines()
self.__videoStat[dname] = dt[0].split(',')
# vhash: [NumofChunks, NumofLeftoverPieces]
self.__mediaReaders[dname] = []
else:
print videopath # not in list, to be deleted
shutil.rmtree(videopath)
else:
self.__MaxNumConn = MAX_PEER_CONNECTION
# need to implement NAT traversal
##########start here############
self.__Addr = (([
ip for ip in gethostbyname_ex(gethostname())[2]
if not ip.startswith("127.")
][0]), choice(range(50000, 60000)))
##########end here############
dirList = os.listdir(self.__PATH)
for dname in dirList:
if os.path.isdir(self.__PATH + dname):
videopath = self.__PATH + dname + os.sep
self.__videoDict[dname] = videopath
self.__mediaReaders[dname] = []
self.__mediaUpdaters[dname] = MediaFileWrapper(
videopath, False) # write access
self.__storeAlloc[dname] = 0
self.__storePrices[dname] = 0
self.__ClientNeighborhood = []
self.__NumofConnections = 0
self.__ClientConnectionIPs = []
self.__ClientConnectionSocks = []
self.__choketimer = 30
self.__uploadBW = 0 # KB/sec
self.__uploadedBytes = 0 # uploaded bytes in a second
self.__UploadThreads = [
] # upload threads. used to control UploadBWControl
self.__neighborLock = thread.allocate_lock()
self.__connectionLock = thread.allocate_lock()
self.__timerLock = thread.allocate_lock()
self.__rateAllocLock = thread.allocate_lock()
self.__mediaReaderLock = thread.allocate_lock()
self.__uploadStatLock = thread.allocate_lock()
self.__UploadThreadsLock = thread.allocate_lock()
self.__BWControlBoot = threading.Event()
self.__BWControlBoot.clear()
# we don't need BW control all the time
self.__uploadBWExceed = threading.Event()
self.__uploadBWExceed.clear()
self.__RateAllocBoot = threading.Event()
self.__RateAllocBoot.clear()
# we don't need rate allocation all the time
def boot(self, uploadBW):
self.__uploadBW = uploadBW # force a virtual upload bandwidth; for testing reasons
if VERBOSE:
print 'Server %s goes online at %s' % (self.__Addr, ctime())
####### This is fake, just for testing purpose ################
t = MyThread(self.__uploadBWControl, (1, ),
self.__uploadBWControl.__name__)
t.start()
sleep(PROCESS_TIME)
# rate allocation
if not self.__isserver:
t = MyThread(self.__rateallocation, (1, ),
self.__rateallocation.__name__)
t.start()
sleep(PROCESS_TIME)
# register to the tracker
t = MyThread(self.__commtotracker, (1, ),
self.__commtotracker.__name__)
t.start()
sleep(PROCESS_TIME)
# listen to passive connections
t = MyThread(self.__WaitforConnection, (1, ),
self.__WaitforConnection.__name__)
t.start()
sleep(PROCESS_TIME)
# actively connect to client
t = MyThread(self.__Connect2Client, self.__MaxNumConn,
self.__Connect2Client.__name__)
t.start()
sleep(PROCESS_TIME)
# choking
t = MyThread(self.__choke, (1, ), self.__choke.__name__)
t.start()
sleep(PROCESS_TIME)
# user prompt
t = MyThread(
self.__userprompt, (1, ),
self.__userprompt.__name__) # open a new thread for new requests
t.start()
sleep(PROCESS_TIME)
# This is an artificial upload BW control mechanism; only used for debugging purpose
def __uploadBWControl(self, *targs):
checkInterval = 0.1
while True:
self.__BWControlBoot.wait()
# wait till there is at least one upload thread running
self.__uploadStatLock.acquire()
exceededBytesPerInterval = self.__uploadedBytes - checkInterval * self.__uploadBW
self.__uploadStatLock.release()
if exceededBytesPerInterval > 0:
self.__uploadBWExceed.clear()
## fake wait to get more upload BW
print 'exceeded'
sleep(exceededBytesPerInterval / self.__uploadBW)
print 'sleep over'
# if there does not exist excessive bytes, the uploadBW will expire
#print self.__uploadedBytes
self.__uploadStatLock.acquire()
self.__uploadedBytes = 0
self.__uploadStatLock.release()
self.__uploadBWExceed.set()
sleep(checkInterval) # check every second
self.__UploadThreadsLock.acquire()
ThreadstoRemove = []
for eachThread in self.__UploadThreads:
if not eachThread.isAlive():
ThreadstoRemove.append(eachThread)
for eachRemoval in ThreadstoRemove:
self.__UploadThreads.remove(eachRemoval)
if len(self.__UploadThreads) == 0:
self.__BWControlBoot.clear(
) # if no threads are running, stop uploadBW control thread
self.__UploadThreadsLock.release()
return
def __choke(self, *targs):
while True:
self.__connectionLock.acquire()
if self.__NumofConnections >= self.__MaxNumConn:
## re-code the choking algorithm by allocating the probabilities##
## START ##
sockind = choice(range(self.__NumofConnections - 1))
## END ##
self.__ClientConnectionSocks[sockind].close()
del self.__ClientConnectionSocks[sockind]
del self.__ClientConnectionIPs[sockind]
self.__NumofConnections -= 1
self.__connectionLock.release()
if self.__NumofConnections < self.__MaxNumConn:
self.__Connect2Client(1) # connect to a new peer
self.__timerLock.acquire()
sleep(self.__choketimer)
self.__timerLock.release()
def __upload(self, *targs):
args = targs[0]
myClisock = args[0]
cliAddr = args[1]
ack = True
avaPieces = None
ClientFile = None
while True:
self.__BWControlBoot.set()
self.__RateAllocBoot.set()
data = myClisock.recvmsg(BUFSIZ)
print data
if data[0] == 'start':
videohash = data[1] # receive movie hash name
if not videohash in self.__videoDict.keys(
): # this will happen for helpers only. Server needs all videos
print self.__videoDict
os.makedirs(self.__PATH + videohash)
self.__videoDict[
videohash] = self.__PATH + videohash + os.sep
self.__mediaReaderLock.acquire()
self.__mediaReaders[videohash] = []
self.__mediaReaderLock.release()
if not self.__isserver:
self.__mediaUpdaters[videohash] = MediaFileWrapper(
self.__videoDict[videohash], False) # write access
self.__rateAllocLock.acquire()
self.__storeAlloc[videohash] = 0
self.__storePrices[videohash] = 0
self.__rateAllocLock.release()
ClientFile = MediaFileWrapper(self.__videoDict[videohash])
self.__mediaReaders[videohash].append(
ClientFile) # add the file reader
myClisock.sendmsg('ACK') # send ack
elif data[0] == 'piece':
if not self.__isserver:
if avaPieces != None:
if len(avaPieces) > 0: # more video than needed
self.__rateAllocLock.acquire()
self.__storePrices[videohash] -= 1 # minus a token
print self.__storePrices
self.__rateAllocLock.release()
else:
pass
avaPieces = ClientFile.getPieceIdbyChunk(data[1])
ack = myClisock.sendmsg(avaPieces)
print avaPieces
elif data[0] == 'content':
ChunkID, PieceID = data[1], data[2]
datamsg = ClientFile.getPiecebyId(ChunkID, PieceID)
if datamsg != None: # found the piece
ack = myClisock.sendmsg('OK')
## this is a fake constraint on upBW; only for testing purpose
self.__uploadBWExceed.wait()
##############################################################
ack = myClisock.sendmsg(datamsg, True)
self.__uploadStatLock.acquire()
self.__uploadedBytes += BYTES_PER_PIECE
self.__uploadStatLock.release()
avaPieces = avaPieces.lstrip(
PieceID) # remove the piece that is uploaded
else:
ack = myClisock.sendmsg('XX') # send out a none message
elif data[0] == 'request':
if not self.__isserver:
if avaPieces != None:
if len(avaPieces) == 0: # could have more video
self.__rateAllocLock.acquire()
self.__storePrices[videohash] += 1 # plus a token
print self.__storePrices
self.__rateAllocLock.release()
if not ack or data == 'closed':
self.__pruneconnections(myClisock, cliAddr)
if ClientFile != None:
ClientFile.closeAll()
self.__mediaReaderLock.acquire()
self.__mediaReaders[videohash].remove(ClientFile)
self.__mediaReaderLock.release()
break
def __download(self, vhashname, segID, pieceIDs):
helper = MediaClient(self.__cwd)
Success = helper.bootashelper(vhashname, segID, pieceIDs, self.__Addr)
del helper
return Success
def __rateallocation(self, *targs):
while True:
self.__RateAllocBoot.wait()
self.__rateAllocLock.acquire()
if len(self.__storeAlloc) != 0:
totalsiz = 0
for vhash in self.__storeAlloc.keys():
self.__storeAlloc[vhash] = getFolderSize(
self.__videoDict[vhash])
totalsiz += self.__storeAlloc[
vhash] # calculate the current total size
pricevideos = zip(self.__storePrices.values(),
self.__storePrices.keys())
pricevideos.sort()
sortedValues, sortedVideos = zip(*pricevideos)
low = 0
high = len(sortedValues) - 1
self.__rateAllocLock.release()
while low <= high:
if totalsiz < STORAGE_CAP:
if sortedValues[high] > 0:
totalsiz += self.__storageupdate(
sortedVideos[low],
totalsiz) # ++ storage of high
high -= 1
continue
else:
break
# no update because not needed
if (abs(
sortedValues[low]
) < ST_ALLOC_TH # no update because it does not satisfy threshold
or abs(sortedValues[high]) < ST_ALLOC_TH):
break
if self.__mediaUpdaters[
sortedVideos[low]].getNumofSeg() <= 0:
low += 1
continue
if self.__mediaUpdaters[
sortedVideos[high]].getNumofSeg() >= SEG_PER_CHUNK:
high -= 1
continue
if sortedValues[low] < 0:
totalsiz += self.__storageupdate(
sortedVideos[low], totalsiz,
False) # -- storage of low
if sortedValues[high] > 0:
totalsiz += self.__storageupdate(
sortedVideos[low], totalsiz)
# ++ storage of high
self.__rateAllocLock.acquire()
for vhash in self.__storePrices.keys():
self.__storePrices[vhash] = 0 # clear
#self.__rateAllocLock.release();
self.__rateAllocLock.release()
sleep(INTERVAL_RATEALLOC)
# don't need rate allocation when no download request is present
self.__UploadThreadsLock.acquire()
ThreadstoRemove = []
for eachThread in self.__UploadThreads:
if not eachThread.isAlive():
ThreadstoRemove.append(eachThread)
for eachRemoval in ThreadstoRemove:
self.__UploadThreads.remove(eachRemoval)
if len(self.__UploadThreads) == 0:
self.__RateAllocBoot.clear(
) # if no threads are running, stop rate allocation thread
self.__UploadThreadsLock.release()
# perform the storage add or removal given a video name
def __storageupdate(self, vhashname, totalsiz, added=True):
mul = 1
if added:
pieceIDs = self.__mediaUpdaters[vhashname].getNewPieceIDs()
segID = self.__mediaUpdaters[vhashname].getNumofSeg() + 1
rsize = self.__videoStat[vhashname][
0] * BYTES_PER_CHUNK / SEG_PER_CHUNK
if rsize + totalsiz < STORAGE_CAP: # write if not exceed storage limit
if not self.__download(vhashname, segID,
pieceIDs): # download data
rsize = 0
else:
rsize = 0
else:
mul = -1
# the update process is self-locked because the MediaFileWrapper itself is a synchronized process
UpdateSuccess = True
self.__mediaReaderLock.acquire()
for eachClientFile in self.__mediaReaders[vhashname]:
UpdateSuccess = UpdateSuccess and eachClientFile.UpdateAccess(
added) # update file object for each peer
self.__mediaReaderLock.release()
self.__mediaUpdaters[vhashname].UpdateAccess(
added) # update file access
if not added and UpdateSuccess: # wait for close() before removal
rsize = self.__mediaUpdaters[vhashname].removeSegment()
return rsize * mul
def __commtotracker(self, *targs):
ack = False
while not ack:
ack = self.__tcpTracSock.Connect2Server(TRACKER_ADDRESS)
if ack:
countdown = 0
while True:
if countdown == 0:
ack = self.__tcpTracSock.sendmsg('serv+' +
repr(self.__Addr[1]))
newneighborhood = self.__tcpTracSock.recvmsg(BUFSIZ)
avaiMovieList = self.__tcpTracSock.recvmsg(BUFSIZ)
if (
not ack
) or newneighborhood == 'closed' or avaiMovieList == 'closed':
ack = False
break
ACK = 'ACK'
if self.__isserver:
movienames = ""
for fhash, fname in self.__movieList.iteritems():
movienames = (movienames + '+' + fname + '#' +
self.__videoStat[fhash][0] +
'#' + self.__videoStat[fhash][1])
ack = self.__tcpTracSock.sendmsg('list' +
movienames)
ACK = self.__tcpTracSock.recvmsg(BUFSIZ)
if (not ack) or ACK == 'closed':
ack = False
break
else:
del self.__videoStat
self.__videoStat = {}
for eachmovie in avaiMovieList.keys():
self.__videoStat[hashlib.md5(
eachmovie).hexdigest(
)] = avaiMovieList[eachmovie]
self.__neighborLock.acquire()
self.__ClientNeighborhood = union(
self.__ClientNeighborhood, newneighborhood)
## might have to set a maximum limit of neighborhood ##
self.__neighborLock.release()
countdown = OBTAIN_NEIGHBOR_PERIOD / INTERVAL_TRACKER_COMMUNICATION
else:
ack = self.__tcpTracSock.sendmsg('alive')
if not ack:
break
sleep(INTERVAL_TRACKER_COMMUNICATION)
countdown -= 1
sleep(TRY_INTERVAL)
del self.__tcpTracSock
self.__tcpTracSock = Mytcpsocket()
def __Connect2Client(self, number):
self.__connectionLock.acquire()
self.__neighborLock.acquire()
Neighborhood = lminus(self.__ClientNeighborhood,
self.__ClientConnectionIPs)
potentialconnect = sample(Neighborhood, min(len(Neighborhood), number))
for eachClient in potentialconnect:
if self.__NumofConnections < self.__MaxNumConn:
if eachClient not in self.__ClientConnectionIPs:
cliSock = Mytcpsocket()
ack1 = cliSock.Connect2Server(eachClient)
ack2 = cliSock.sendmsg(self.__Addr[1])
ack3 = cliSock.recvmsg(BUFSIZ) # receive ACK
if ack1 and ack2 and ack3 != 'closed':
self.__NumofConnections += 1
self.__ClientConnectionSocks.append(cliSock)
self.__ClientConnectionIPs.append(eachClient)
t = MyThread(self.__upload, (cliSock, eachClient),
self.__upload.__name__)
self.__UploadThreadsLock.acquire()
self.__UploadThreads.append(t)
self.__UploadThreadsLock.release()
self.__BWControlBoot.set()
# start BW control
self.__RateAllocBoot.set()
# start rate allocation
t.start()
sleep(PROCESS_TIME)
else:
self.__ClientNeighborhood.remove(eachClient)
else:
break
self.__connectionLock.release()
self.__neighborLock.release()
def __WaitforConnection(self, *targs):
self.__tcpSerSock.InitServSock(self.__Addr, self.__MaxNumConn)
if VERBOSE:
print "Waiting for connection..."
while True:
(tcpClientSock, CLIENT_ADDR) = self.__tcpSerSock.WaitforConn()
self.__connectionLock.acquire()
clitcpsock = Mytcpsocket(tcpClientSock)
if self.__NumofConnections < self.__MaxNumConn:
servport = clitcpsock.recvmsg(BUFSIZ)
ack = clitcpsock.sendmsg('ACK')
CLIENT_SERV_ADDR = (CLIENT_ADDR[0], servport)
if ((CLIENT_SERV_ADDR in self.__ClientConnectionIPs)
or (not ack) or servport == 'closed'):
clitcpsock.close()
else:
self.__ClientConnectionSocks.append(clitcpsock)
self.__ClientConnectionIPs.append(CLIENT_SERV_ADDR)
self.__NumofConnections += 1
if VERBOSE:
print "...connected from:", CLIENT_ADDR
t = MyThread(self.__upload, (clitcpsock, CLIENT_SERV_ADDR),
self.__upload.__name__)
self.__UploadThreadsLock.acquire()
self.__UploadThreads.append(t)
self.__UploadThreadsLock.release()
self.__BWControlBoot.set()
# start BW control
self.__RateAllocBoot.set()
# start rate allocation
t.start()
sleep(PROCESS_TIME)
else:
clitcpsock.close(
) # close if the number of connections exceeds max
self.__connectionLock.release()
# no random coding of the pieces yet. need to modify the function if needed to do so.
def __convertVideo(self, path, fname):
FILES = []
METAS = []
vidfile = open(fname, 'rb')
for segNum in range(SEG_PER_CHUNK):
file = open(path + 'seg' + repr(segNum + 1) + FILE_SUFFIX, 'wb+')
meta = open(path + 'seg' + repr(segNum + 1) + META_SUFFIX, 'wb+')
FILES.append(file)
METAS.append(meta)
Ind = 0
data = vidfile.read(BYTES_PER_SEG)
numofchunks = 0
numofleftpieces = 0
while data != "":
wlen = len(data) * 1.0 / BYTES_PER_PIECE
if wlen < PIECE_PER_SEG:
wlen = ceil(wlen)
data = pack(str(int(wlen * BYTES_PER_PIECE)) + 's', data)
numofleftpieces += wlen
FILES[Ind].write(data)
METAS[Ind].write(
PIECE_IDS[Ind * PIECE_PER_SEG:Ind * PIECE_PER_SEG + int(wlen)])
Ind += 1
Ind = Ind % SEG_PER_CHUNK
data = vidfile.read(BYTES_PER_SEG)
vidfile.close()
numofchunks = int(numofleftpieces / PIECE_PER_CHUNK)
numofleftpieces = int(numofleftpieces % PIECE_PER_CHUNK)
if numofleftpieces == 0:
numofleftpieces = PIECE_PER_CHUNK
f = open(path + 'length', 'wb+')
f.write(str(numofchunks) + ',' + str(numofleftpieces))
f.close()
for segNum in range(SEG_PER_CHUNK):
FILES[segNum].close()
METAS[segNum].close()
def __userprompt(self, *args):
while True:
userinput = raw_input(
'Input your choice:\n[1] Print Neighbors\n[2] Print Connections\nYour choice>> '
)
if userinput == '1':
self.__printAllNeighbors()
if userinput == '2':
self.__printAllConnections()
else:
pass
def __pruneconnections(self, tcpsock, Addr):
self.__connectionLock.acquire()
self.__ClientConnectionSocks.remove(tcpsock)
self.__ClientConnectionIPs.remove(Addr)
self.__NumofConnections -= 1
self.__connectionLock.release()
def __printAllNeighbors(self):
for eachNeighbor in self.__ClientNeighborhood:
print eachNeighbor
def __printAllConnections(self):
for eachConnectionIP in self.__ClientConnectionIPs:
print eachConnectionIP
for eachConnectionSock in self.__ClientConnectionSocks:
print eachConnectionSock
class MediaClient(object):
'# Author: Hao Zhang\
# Created: 02/11/2011\
# Class: MediaClient - a TCP client that downloads data from the servers'
# Initialization -- set up address and port; initilize parameters
def __init__(self, cwd):
self.__cwd = cwd # current working directory
self.__BootedasUser = True
self.__tcpTracSock = Mytcpsocket()
self.__tcpListenSock = Mytcpsocket()
self.__MaxNumConn = MAX_PEER_CONNECTION
self.__Addr = (([
ip for ip in gethostbyname_ex(gethostname())[2]
if not ip.startswith("127.")
][0]), choice(range(50000, 60000)))
self.__tcpCliSockets = []
self.__ServerNeighborhood = []
self.__ServerConnectionIPs = []
self.__ServerConnectionSocks = []
self.__NumofConnections = 0
self.__ExcludedServerAddr = []
self.__choketimer = 30
self.__StartWaiting = False
self.__receivedFile = None # streaming file object
self.__avaiMovieList = {} # available movie list from the tracker
self.__movieHashNames = {}
self.__currentvid = 'None' # current video that is being watched
self.__NumofChunks = 0
self.__NumofLeftPieces = 0
self.__neighborLock = thread.allocate_lock(
) # lock for the neighborhood update
self.__connectionLock = thread.allocate_lock()
self.__timerLock = thread.allocate_lock()
self.__streamingStatLock = thread.allocate_lock()
self.__bufferStatLock = thread.allocate_lock()
self.__emptyAvailable = threading.Event()
self.__emptyAvailable.clear()
# one of the download chunks is waiting for other corresponding threads to finish that chunk
self.__DownloadWait = threading.Event()
self.__DownloadWait.clear()
# wait till download is finished
self.__BufferFullWait = threading.Event()
self.__BufferFullWait.clear()
# buffer too much, wait before download
self.__BufferEmptyWait = threading.Event()
self.__BufferEmptyWait.clear()
# buffer not enough, download from server
self.__streamingWait = threading.Event()
self.__streamingWait.clear()
# wait till buffer is ready to stream
self.__downloadthreads = []
self.__EXIT = False
# Boot as a pure user with prompt
def bootasuser(self):
self.__streamingPath = self.__cwd + os.sep + CLIENT_PATH + os.sep
if not os.path.exists(self.__streamingPath):
os.makedirs(self.__streamingPath)
self.__PiecestoDownload = PIECE_IDS # as a user, download all the pieces
self.__PiecesperChunk = PIECE_PER_CHUNK
if VERBOSE:
print 'User %s goes online at %s' % (self.__Addr, ctime())
# register to the tracker
t = MyThread(self.__commtotracker, (1, ),
self.__commtotracker.__name__)
t.start()
sleep(PROCESS_TIME)
# choking
t = MyThread(self.__choke, (1, ), self.__choke.__name__)
t.start()
sleep(PROCESS_TIME)
# enable user prompt
t = MyThread(
self.__userprompt, (1, ),
self.__userprompt.__name__) # open a new thread for new requests
t.start()
sleep(PROCESS_TIME)
# Boot as a downloader to get packets (as a helper)
def bootashelper(self, videohash, segID, piecestodownload,
excludedServerAddr):
self.__BootedasUser = False
self.__ExcludedServerAddr = excludedServerAddr
self.__PiecestoDownload = piecestodownload # as a helper, specify pieces to download
self.__PiecesperChunk = len(piecestodownload)
self.__streamingPath = self.__cwd + os.sep + SERVER_PATH + os.sep + videohash + os.sep + 'seg' + str(
segID) + FILE_SUFFIX
self.__MetaPath = self.__cwd + os.sep + SERVER_PATH + os.sep + videohash + os.sep + 'seg' + str(
segID) + META_SUFFIX
self.__BufferFullWait.set() # no need to buffer download for helper
#### need a mechanism to control this: need to be either too greedy or too conservative
self.__BufferEmptyWait.set()
######
# register to the tracker
t = MyThread(self.__commtotracker, (1, ),
self.__commtotracker.__name__)
t.start()
sleep(PROCESS_TIME * 2)
# choking
t = MyThread(self.__choke, (1, ), self.__choke.__name__)
t.start()
sleep(PROCESS_TIME)
# enable download
self.__movieInitialize(videohash)
sleep(PROCESS_TIME)
self.__DownloadWait.wait()
# wait until download is finished
self.__EXIT = True
return True
def __bufferControl(self, *targs):
while True:
if self.__EXIT:
return
self.__streamingStatLock.acquire()
if self.__streamedNumofChunks >= self.__NumofChunks:
print self.__NumofChunks, self.__streamedNumofChunks
self.__streamingWait.set()
# let the stream break
self.__streamingStatLock.release()
break
if self.__streamingBufferLength < 1: # less than a second of buffer to play
self.__streamingWait.clear() # stop streaming
self.__BufferEmptyWait.set()
self.__BufferFullWait.set()
# start downloading NOW!
else:
self.__streamingWait.set() # let it stream
if self.__streamingBufferLength >= BUFFER_TOO_MUCH: # buffer too much
self.__BufferFullWait.clear()
self.__BufferEmptyWait.clear() # stop downloading
elif self.__streamingBufferLength >= BUFFER_LENGTH:
self.__BufferEmptyWait.clear()
elif self.__streamingBufferLength < BUFFER_LENGTH: # buffer less than required length
self.__BufferFullWait.set() # download from helpers
if self.__streamingBufferLength < BUFFER_EMERG_LEFT: # if less than emergency
self.__BufferEmptyWait.set(
) # download from the SERVER
else:
self.__BufferEmptyWait.clear()
self.__streamingStatLock.release()
sleep(BUFFER_CHECK_INTERVAL)
def __streaming(self, *targs):
speedup = 1
while True:
if self.__EXIT:
return
self.__streamingWait.wait()
### stream the video NOW for duration xxx ###
### Now it is Fake streaming ###
sleep(1)
self.__streamingStatLock.acquire()
if self.__streamingBufferLength > 1:
self.__streamingBufferLength -= speedup
print 'subtracted, BufferLength = ', self.__streamingBufferLength
self.__streamedNumofChunks += speedup * self.__streamingRate / BYTES_PER_CHUNK
if self.__streamedNumofChunks >= self.__NumofChunks:
print self.__NumofChunks, self.__streamedNumofChunks
self.__streamingStatLock.release()
break
self.__streamingStatLock.release()
################################
def __choke(self, *targs):
while True:
if self.__EXIT:
return
self.__connectionLock.acquire()
if self.__NumofConnections >= self.__MaxNumConn:
## re-code the choking algorithm by allocating the probabilities##
## START ##
sockind = choice(range(self.__NumofConnections - 1))
## END ##
self.__ServerConnectionSocks[sockind].close()
del self.__ServerConnectionSocks[sockind]
del self.__ServerConnectionIPs[sockind]
self.__NumofConnections -= 1
self.__connectionLock.release()
if self.__NumofConnections < self.__MaxNumConn and self.__currentvid != 'None':
self.__Connect2Server(
(1, self.__currentvid)) # connect to a new peer
self.__timerLock.acquire()
sleep(self.__choketimer)
self.__timerLock.release()
def __download(self, *targs):
args = targs[0]
servSock = args[0]
servAddr = args[1]
videohash = args[2]
self.__DownloadWait.clear()
while True:
if videohash == 'None':
servSock.sendmsg('closed')
self.__pruneconnections(servSock, servAddr)
self.__DownloadWait.set()
return
ack = servSock.sendmsg(['start',
videohash]) # send which video to watch
ACK = servSock.recvmsg(BUFSIZ) # receive acknowledgement
if not ack or ACK == 'closed':
self.__pruneconnections(servSock, servAddr)
self.__DownloadWait.set()
return
currentChunkID = -1
piecesperchunk = self.__PiecesperChunk
while True:
if self.__EXIT:
return
if servAddr == MEDIA_SERVER_ADDRESS: # wait until buffer is too little
self.__BufferEmptyWait.wait()
else: # wait until buffer is less than a certain threshold
self.__BufferFullWait.wait()
self.__bufferStatLock.acquire()
if videohash != self.__currentvid:
videohash = self.__currentvid
self.__bufferStatLock.release()
break
if currentChunkID < self.__bufferheadChunkID:
currentChunkID = self.__bufferheadChunkID
self.__bufferStatLock.release()
ack = servSock.sendmsg(['piece', currentChunkID])
#print ack
avaiPieceIDsfromServ = servSock.recvmsg(BUFSIZ)
#print avaiPieceIDsfromServ
if not ack or avaiPieceIDsfromServ == 'closed':
self.__pruneconnections(servSock, servAddr)
self.__DownloadWait.set()
return
continue
avaiPieceIDsfromServ = intersect(
avaiPieceIDsfromServ, self.__avaiPieceIDs
) # Now I am sure no other peers are giving me the same data
RequestingSet = lminus(avaiPieceIDsfromServ,
self.__PiecesNowRequested)
#print RequestingSet, "from", servAddr
if len(RequestingSet) == 0:
if len(
lminus(self.__avaiPieceIDs,
self.__PiecesNowRequested)) > 0:
ack = servSock.sendmsg(['request'])
if not ack:
self.__pruneconnections(servSock, servAddr)
self.__bufferStatLock.release()
self.__DownloadWait.set()
return
self.__bufferStatLock.release()
self.__emptyAvailable.wait(
BUFFER_EMERG_LEFT) # wait until this chunk is over
continue
PiecesToRequest = choice(
RequestingSet) # randomly choose an available piece
self.__PiecesNowRequested.extend(
PiecesToRequest) # put it into the requesting queue
self.__bufferStatLock.release()
ack = servSock.sendmsg(
['content', currentChunkID,
PiecesToRequest]) # download the piece
ACK = servSock.recvmsg(BUFSIZ)
if ACK == 'OK':
data = servSock.recvmsg(
BYTES_PER_PIECE,
True) # only receive if server still has it
else: # if ACK == 'XX'
avaiPieceIDsfromServ.remove(
PiecesToRequest) # otherwise remove it and continue
self.__PiecesNowRequested = lminus(
self.__PiecesNowRequested, PiecesToRequest)
continue
#print data[0:5]
print "data from server", servAddr
self.__bufferStatLock.acquire()
if videohash != self.__currentvid: # if no longer watching current video, leave
videohash = self.__currentvid
self.__bufferStatLock.release()
break
if not ack or data == 'closed': # check if the connection is still alive
self.__pruneconnections(servSock, servAddr)
self.__PiecesNowRequested = lminus(
self.__PiecesNowRequested, PiecesToRequest)
self.__bufferStatLock.release()
self.__DownloadWait.set()
return
if currentChunkID < self.__bufferheadChunkID: # if other processes have already advanced the chunk, then dump it
self.__bufferStatLock.release()
continue
# finally I made sure the downloaded pieces is valid and needed
self.__PiecesNowRequested = lminus(
self.__PiecesNowRequested,
PiecesToRequest) # change status requested to available
self.__avaiPieceIDs = lminus(self.__avaiPieceIDs,
PiecesToRequest)
self.__bufferPieceIDs.extend(PiecesToRequest)
self.__bufferContent.append(data)
if len(self.__bufferPieceIDs
) >= piecesperchunk: # finished downloading of a chunk
sortedIDs, self.__bufferContent = PacketsDecode(
self.__bufferContent, self.__bufferPieceIDs)
if self.__BootedasUser:
##### feed the buffer to the streaming wrapper #####
##### now just write the file #####
self.__receivedFile.write(self.__bufferContent)
##### END #########################
else:
self.__MetaFile.write(sortedIDs)
self.__receivedFile.write(self.__bufferContent)
self.__streamingStatLock.acquire()
# change streaming buffer length in seconds
self.__streamingBufferLength += piecesperchunk * BYTES_PER_PIECE / self.__streamingRate
print 'added, BufferLength = ', self.__streamingBufferLength
self.__streamingStatLock.release()
self.__bufferPieceIDs = []
self.__bufferContent = []
self.__PiecesNowRequested = []
self.__avaiPieceIDs = self.__PiecestoDownload
piecesperchunk = self.__PiecesperChunk
self.__bufferheadChunkID += 1
if self.__bufferheadChunkID == self.__NumofChunks:
self.__avaiPieceIDs = intersect(
PIECE_IDS[0:self.__NumofLeftPieces],
self.__PiecestoDownload) # last chunk left pieces
piecesperchunk = len(self.__avaiPieceIDs)
if self.__bufferheadChunkID > self.__NumofChunks or piecesperchunk == 0:
self.__currentvid = 'None' # finish download
self.__NumofChunks = 0
self.__receivedFile.close()
if not self.__BootedasUser:
self.__MetaFile.close()
self.__DownloadWait.set()
# set the wait to finish
self.__emptyAvailable.set()
# release the available ID empty lock
self.__emptyAvailable.clear()
# set the lock again for next round
self.__bufferStatLock.release()
def __movieInitialize(self, videohash):
## dump all current # dump the current streaming if possible
self.__bufferStatLock.acquire()
if self.__currentvid == videohash:
self.__bufferStatLock.release()
return
#### movie paramter initialization:
self.__streamingRate = BYTES_PER_CHUNK / BUFFER_LENGTH
self.__streamingBufferLength = 0 # in units of seconds
self.__streamedNumofChunks = 0 # number of chunks that are streamed
self.__bufferheadChunkID = 0 # the chunk ID in front of the buffer
self.__bufferContent = [] # integer IDs
self.__bufferPieceIDs = [] # string names
self.__PiecesNowRequested = [] # pieces that are now requested
self.__avaiPieceIDs = self.__PiecestoDownload # initialize available pieces to watch
self.__streamingWait.clear()
self.__BufferFullWait.clear()
self.__BufferEmptyWait.clear()
if videohash != 'None': # if a movie was requested
self.__NumofChunks = self.__movieHashNames[videohash][0]
self.__NumofLeftPieces = self.__movieHashNames[videohash][1]
self.__currentvid = videohash
if self.__receivedFile != None: # close the opened file
self.__receivedFile.close()
if self.__BootedasUser:
writeFilePath = self.__streamingPath + videohash + '.flv'
#writeFilePath = self.__streamingPath + 'hancock-tsr2_h480p' + '.flv'
else:
writeFilePath = self.__streamingPath
self.__MetaFile = open(self.__MetaPath, 'w+b')
self.__receivedFile = open(writeFilePath,
'w+b') # open a writable file
if self.__BootedasUser:
t = MyThread(self.__bufferControl, (1, ),
self.__bufferControl.__name__)
t.start()
sleep(PROCESS_TIME)
# start buffer control
t = MyThread(self.__streaming, (1, ),
self.__streaming.__name__)
# start streaming
t.start()
sleep(PROCESS_TIME)
else:
self.__BufferEmptyWait.set()
self.__BufferFullWait.set()
t = MyThread(self.__Connect2Server, (self.__MaxNumConn, videohash),
self.__Connect2Server.__name__)
t.start()
sleep(PROCESS_TIME)
if self.__StartWaiting == False: # listen to active server to user connections
self.__StartWaiting = True
t = MyThread(self.__WaitforConnection, (1, ),
self.__WaitforConnection.__name__)
t.start()
sleep(PROCESS_TIME)
else:
self.__NumofChunks = 0
self.__NumofLeftPieces = 0
self.__currentvid = 'None' # not watching any videos
self.__bufferStatLock.release()
def __commtotracker(self, *targs):
ack = False
while not ack:
if self.__EXIT:
return
ack = self.__tcpTracSock.Connect2Server(TRACKER_ADDRESS)
#print TRACKER_ADDRESS
if ack:
countdown = 0
while True:
if self.__EXIT:
return
if countdown == 0:
ack = self.__tcpTracSock.sendmsg('user+' +
repr(self.__Addr[1]))
newneighborhood = self.__tcpTracSock.recvmsg(BUFSIZ)
self.__avaiMovieList = self.__tcpTracSock.recvmsg(
BUFSIZ)
if (
not ack
) or newneighborhood == 'closed' or self.__avaiMovieList == 'closed':
ack = False
break
del self.__movieHashNames
self.__movieHashNames = {}
for eachmovie in self.__avaiMovieList.keys():
self.__movieHashNames[hashlib.md5(
eachmovie).hexdigest(
)] = self.__avaiMovieList[eachmovie]
self.__neighborLock.acquire()
## might have to set a maximum limit of neighborhood ##
self.__ServerNeighborhood = union(
self.__ServerNeighborhood, newneighborhood)
self.__ServerNeighborhood = lminus(
self.__ServerNeighborhood,
self.__ExcludedServerAddr)
self.__neighborLock.release()
countdown = OBTAIN_NEIGHBOR_PERIOD / INTERVAL_TRACKER_COMMUNICATION
else:
ack = self.__tcpTracSock.sendmsg('alive')
if not ack:
break
sleep(INTERVAL_TRACKER_COMMUNICATION)
countdown -= 1
# send alive msg interval
sleep(TRY_INTERVAL) # obtain neighborhood/movielist interval
del self.__tcpTracSock
self.__tcpTracSock = Mytcpsocket()
def __WaitforConnection(self, *targs):
self.__tcpListenSock.InitServSock(self.__Addr, self.__MaxNumConn)
while True:
if self.__EXIT:
return
(tcpServSock, SERVER_ADDR) = self.__tcpListenSock.WaitforConn()
self.__connectionLock.acquire()
servtcpsock = Mytcpsocket(tcpServSock)
if self.__NumofConnections < self.__MaxNumConn:
servport = servtcpsock.recvmsg(BUFSIZ)
ack = servtcpsock.sendmsg('ACK') # send ACK
SERVER_SERV_ADDR = (SERVER_ADDR[0], servport)
if ((SERVER_SERV_ADDR in self.__ServerConnectionIPs)
or (SERVER_SERV_ADDR == self.__ExcludedServerAddr)
or (not ack) or servport == 'closed'):
servtcpsock.close()
else:
self.__ServerConnectionSocks.append(servtcpsock)
self.__ServerConnectionIPs.append(SERVER_SERV_ADDR)
self.__NumofConnections += 1
if VERBOSE:
print "...connected from:", SERVER_SERV_ADDR
t = MyThread(
self.__download,
(servtcpsock, SERVER_SERV_ADDR, self.__currentvid),
self.__download.__name__)
t.start()
sleep(PROCESS_TIME)
self.__downloadthreads.append(t)
else:
servtcpsock.close(
) # close if the number of connections exceeds max
self.__connectionLock.release()
def __Connect2Server(self, *targs):
args = targs[0]
number = args[0]
videohash = args[1]
self.__connectionLock.acquire()
self.__neighborLock.acquire()
Neighborhood = lminus(self.__ServerNeighborhood,
self.__ServerConnectionIPs)
potentialconnect = sample(Neighborhood, min(len(Neighborhood), number))
for eachServer in potentialconnect:
if self.__NumofConnections < self.__MaxNumConn:
if eachServer not in self.__ServerConnectionIPs:
servSock = Mytcpsocket()
ack1 = servSock.Connect2Server(eachServer)
ack2 = servSock.sendmsg(self.__Addr[1])
ack3 = servSock.recvmsg(BUFSIZ) # receive ACK
if ack1 and ack2 and ack3 != 'closed':
self.__NumofConnections += 1
self.__ServerConnectionSocks.append(servSock)
self.__ServerConnectionIPs.append(eachServer)
t = MyThread(self.__download,
(servSock, eachServer, videohash),
self.__download.__name__)
t.start()
sleep(PROCESS_TIME)
self.__downloadthreads.append(t)
else:
self.__ServerNeighborhood.remove(eachServer)
else:
break
self.__connectionLock.release()
self.__neighborLock.release()
def __userprompt(self, *args):
while True:
if self.__EXIT:
return
userinput = raw_input(
'<<<Select a function>>>:\n[-1] Exit\n[1] Watch a Movie\n[2] Print Neighbors\n[3] Print Connections\nYour choice>> '
)
if userinput == '1':
numofmovies = len(self.__avaiMovieList)
if numofmovies == 0:
print 'No movies to watch yet!'
continue
print '<<<Select a movie>>>:'
print '#. <<Go back>>'
print '0. <<Stop the current movie>>'
for i, fname in enumerate(self.__avaiMovieList):
print repr(i + 1) + '. ' + fname
input = raw_input('Your choice>> ')
if input == '#': # if the user wants to go back
pass
elif input >= '0' and input <= str(numofmovies):
movieID = int(input) - 1
if movieID >= 0: # watch a movie
for i, eachmovie in enumerate(self.__avaiMovieList):
if i == movieID:
videohash = hashlib.md5(eachmovie).hexdigest()
break
else:
videohash = 'None' # stop watching the current movie
self.__movieInitialize(
videohash) # initialize movie download and watching
elif userinput == '2':
self.__printAllNeighbors()
elif userinput == '3':
self.__printAllConnections()
elif userinput == '-1':
self.__EXIT = True
return
else:
pass
def __pruneconnections(self, tcpsock, Addr):
self.__connectionLock.acquire()
print tcpsock, Addr
print self.__ServerConnectionSocks
self.__ServerConnectionSocks.remove(tcpsock)
self.__ServerConnectionIPs.remove(Addr)
self.__NumofConnections -= 1
self.__connectionLock.release()
def __closeAllconnections(self):
self.__connectionLock.acquire()
for connection in self.__ServerConnectionSocks:
connection.close()
self.__ServerConnectionSocks = []
self.__ServerConnectionIPs = []
self.__NumofConnections = 0
self.__connectionLock.release()
def __printAllNeighbors(self):
for eachNeighbor in self.__ServerNeighborhood:
print eachNeighbor
def __printAllConnections(self):
for eachConnectionIP in self.__ServerConnectionIPs:
print eachConnectionIP
for eachConnectionSock in self.__ServerConnectionSocks:
print eachConnectionSock