def streamMethod3(self):
chunkIndex = 1
manifest = Manifest("BunnyManifest_1.xml")
bitrates = manifest.listOfBitrates()
# Compute switch-up epsilon
epsilon = self.computeEpsilon(bitrates)
#start with lowest bitrate
bitrateIndex = 0
numChunks = 99 #manifest.numberOfChunks(bitrates[bitrateIndex])
idleThreadIDs = Set()
startNewRound = False
#threadBandwidth = {}
for thread in self.threads:
if thread.successful:
idleThreadIDs.add(thread.threadID)
#threadBandwidth[thread.threadID] = 0
if len(idleThreadIDs) == NUM_THREADS:
startNewRound = True
finished = False
bitrateFreq = {}
for bitrate in bitrates:
bitrateFreq[bitrate] = 0
# Start Player
p = Player(self.buffer)
p.start()
while not finished:
for thread in self.threads:
if not thread.isAlive():
# Check if we have gotten all the chunks
# If so, we are done
if chunkIndex == numChunks + 1:
finished = True
break
# if thread finishes downloading, look for more work
if not startNewRound and thread.successful:
minProgress = None
work = None
videoURL = None
worstThread = None
canHelpNotHelping = False
for currThread in self.threads:
if (currThread.downloadList[0][1] is 0 or currThread.downloadList[0][1] is None) and currThread.isAlive():
canHelpNotHelping = True
break
for currThread in self.threads:
if currThread.isAlive() and (not canHelpNotHelping or (canHelpNotHelping and (currThread.downloadList[0][1] is 0 or currThread.downloadList[0][1] is None))):
size = self.buffer.getFileSize(currThread.bufferIndices[0], currThread.downloadList[0][1], currThread.downloadList[0][2])
# If -1 returned, don't want to help out; just skip
if size == -1:
continue
progress = float(size) / currThread.downloadSize
if minProgress is None or progress < minProgress:
minProgress = progress
work = (currThread.downloadSize - size) / 2
videoURL = currThread.downloadList[0][0]
worstThread = currThread
# if all threads finish or further work is unnecessary, start new round of downloads
if minProgress is None or work < 15000:
idleThreadIDs.add(thread.threadID)
if len(idleThreadIDs) == NUM_THREADS:
startNewRound = True
else:
#split work
ID = thread.threadID
print "splitting: " + str(ID)
# range request for new thread
videoURL = serverURLs[thread.threadID] + re.findall("[0-9]/.*", videoURL)[0][1:]
size = self.buffer.getFileSize(worstThread.bufferIndices[0], worstThread.downloadList[0][1], worstThread.downloadList[0][2])
# calculate new thread's range
if worstThread.downloadList[0][1] is None:
endByte = worstThread.downloadSize - 1
beginByte = size + ((endByte - size) / 2)
#print "None Case"
#print "(begin, end)", beginByte, endByte
else:
endByte = worstThread.downloadList[0][2]
beginByte = worstThread.downloadList[0][1] + size + ((endByte - (worstThread.downloadList[0][1] + size)) / 2)
#print "Other Case"
#print "(begin, end)", beginByte, endByte
downloadSize = endByte - beginByte + 1
# reduce byte range of helped thread
worstThread.downloadSize -= downloadSize
oldBegin = worstThread.downloadList[0][1]
oldEnd = worstThread.downloadList[0][2]
if oldBegin == None:
newBegin = 0
else:
newBegin = oldBegin
newEnd = beginByte - 1
worstThread.downloadList[0][1] = newBegin
worstThread.downloadList[0][2] = newEnd
index = worstThread.bufferIndices[0]
self.buffer.rewriteBeginAndEnd(index, self.buffer.read(index, oldBegin, oldEnd), oldBegin, oldEnd, newBegin, newEnd)
# notify helped thread that its byte range is reduced
worstThread.reduced = True
if ID in idleThreadIDs:
idleThreadIDs.remove(ID)
self.threads[ID] = Downloader(ID, [[videoURL, beginByte, endByte]], self.buffer, [index], thread.bandwidth, downloadSize, thread.connectionBeginTime)
self.threads[ID].start()
while self.buffer.read(index, beginByte, endByte) is "":
pass
# start new round
if startNewRound and self.buffer.size - self.buffer.length >= NUM_THREADS:
# bitrate adaptation based on bandwidth
# do not do for first round
if chunkIndex != 1:
time = 0
min = None
max = None
for currThread in self.threads:
if min == None or currThread.connectionBeginTime < min:
min = currThread.connectionBeginTime
if max == None or currThread.endTime > max:
max = currThread.endTime
deltaT = max - min
time = deltaT.microseconds + deltaT.seconds * 1000000
u = float(CHUNK_DURATION * NUM_THREADS) / time
print time
print u
if u > 1 + epsilon:
if bitrateIndex != len(bitrates) - 1:
bitrateIndex += 1
elif u < 1:
if bitrateIndex != 0:
bitrateIndex -= 1
#threadBandwidth[thread.threadID] += u
# threads have been assigned to download
idleThreadIDs = Set()
# starting of new round finishes when all threads are downloading
startNewRound = False
# Get threadID's and their threadInfo
threadInfo = []
for currThread in self.threads:
threadInfo.append([currThread.threadID, currThread.bandwidth])
# sort threadInfo so that higher bandwidth connections get assigned
# to earlier byte ranges
threadInfo = sorted(threadInfo, key=lambda list: list[1], reverse = True)
for info in threadInfo:
if chunkIndex == numChunks + 1:
break
ID = info[0]
currThread = self.threads[ID]
# Get url from manifest file based on bitrate and chunk index
URLend, chunkSize = manifest.read(bitrates[bitrateIndex], chunkIndex)
videoURL = serverURLs[ID] + "/" + URLend
self.threads[ID] = Downloader(ID, [[videoURL, None, None]], self.buffer, [chunkIndex], currThread.bandwidth, chunkSize)
self.threads[ID].start()
self.buffer.setSize(chunkIndex, chunkSize)
chunkIndex += 1
bitrateFreq[bitrates[bitrateIndex]] += 1
self.buffer.length += 1
# at every round, plot figure using existing information
statBitrateFreq = {}
statThreadBandwidth = {}
#print "Chunk Index", chunkIndex
#print bitrateFreq
#print threadBandwidth
for key in bitrateFreq.keys():
value = bitrateFreq[key]
statBitrateFreq[key] = (float(value)/chunkIndex) * 100
#for key in threadBandwidth.keys():
# value = threadBandwidth[key]
# statThreadBandwidth[key] = value/chunkIndex
self.plot(statBitrateFreq, "Our Method")
def streamMethod1(self):
chunkIndex = 1
manifest = Manifest("BunnyManifest_1.xml")
bitrates = manifest.listOfBitrates()
# Compute switch-up epsilon
epsilon = self.computeEpsilon(bitrates)
#start with lowest bitrate
bitrateIndex = 0
numChunks = 99 #manifest.numberOfChunks(bitrates[bitrateIndex])
idleThreadIDs = Set()
startNewRound = False
#threadBandwidth = {}
for thread in self.threads:
if thread.successful:
idleThreadIDs.add(thread.threadID)
#threadBandwidth[thread.threadID] = 0
if len(idleThreadIDs) == NUM_THREADS:
startNewRound = True
finished = False
# Used for recording bitrate frequences for graphing
bitrateFreq = {}
for bitrate in bitrates:
bitrateFreq[bitrate] = 0
# Start Player
p = Player(self.buffer)
p.start()
while not finished:
for thread in self.threads:
if not thread.isAlive():
# Check if we have gotten all the chunks
# If so, we are done
if chunkIndex == numChunks + 1:
finished = True
break
# if threads finish after round of downloads started, record their IDs
if not startNewRound and thread.successful:
idleThreadIDs.add(thread.threadID)
# if all threads finish, start new round of downloads
if len(idleThreadIDs) == NUM_THREADS:
startNewRound = True
# start new round
if startNewRound and self.buffer.size - self.buffer.length >= NUM_THREADS:
# bitrate adaptation based on bandwidth
# do not do for first round
if chunkIndex != 1:
time = 0
min = None
max = None
for currThread in self.threads:
if min == None or currThread.beginTime < min:
min = currThread.beginTime
if max == None or currThread.endTime > max:
max = currThread.endTime
deltaT = max - min
time = deltaT.microseconds + deltaT.seconds * 1000000
u = float(CHUNK_DURATION * NUM_THREADS) / time
print time
print u
if u > 1 + epsilon:
if bitrateIndex != len(bitrates) - 1:
bitrateIndex += 1
elif u < 1:
if bitrateIndex != 0:
bitrateIndex -= 1
#threadBandwidth[thread.threadID] += u
# threads have been assigned to download
idleThreadIDs = Set()
# starting of new round finishes when all threads are downloading
startNewRound = False
# Request urls from server
for currThread in self.threads:
ID = currThread.threadID
# Get url from manifest file based on bitrate and chunk index
videoURL, chunkSize = manifest.read(bitrates[bitrateIndex], chunkIndex)
videoURL = serverURLs[ID] + "/" + videoURL
bitrateFreq[bitrates[bitrateIndex]] += 1
self.buffer.setSize(chunkIndex, chunkSize)
self.threads[ID] = Downloader(ID, [[videoURL, None, None]], self.buffer, [chunkIndex])
self.threads[ID].start()
chunkIndex += 1
self.buffer.length += 1
# at every round, plot figure using existing information
statBitrateFreq = {}
statThreadBandwidth = {}
# Convert to percentage and graph
for key in bitrateFreq.keys():
value = bitrateFreq[key]
statBitrateFreq[key] = (float(value)/chunkIndex) * 100
self.plot(statBitrateFreq, "Naive Parallel")
def streamMethod2(self):
chunkIndex = 1
redoTime = 0
manifest = Manifest("BunnyManifest_1.xml")
bitrates = manifest.listOfBitrates()
# Compute switch-up epsilon
epsilon = self.computeEpsilon(bitrates)
#start with lowest bitrate
bitrateIndex = 0
numChunks = 99 #manifest.numberOfChunks(bitrates[bitrateIndex])
idleThreadIDs = Set()
startNewRound = False
#threadBandwidth = {}
for thread in self.threads:
if thread.successful:
idleThreadIDs.add(thread.threadID)
#threadBandwidth[thread.threadID] = 0
if len(idleThreadIDs) == NUM_THREADS:
startNewRound = True
finished = False
bitrateFreq = {}
for bitrate in bitrates:
bitrateFreq[bitrate] = 0
# Start Player
#p = Player(self.buffer)
#p.start()
while not finished:
for thread in self.threads:
if not thread.isAlive():
# Check if we have gotten all the chunks
# If so, we are done
if chunkIndex == numChunks + 1:
finished = True
break
# if threads finish after round of downloads started, record their IDs
if not startNewRound and thread.successful:
idleThreadIDs.add(thread.threadID)
# if all threads finish, start new round of downloads
if len(idleThreadIDs) == NUM_THREADS:
startNewRound = True
else:
# if finished downloading file first, stop duplicate downloads
for currThread in self.threads:
if currThread.isAlive():
# just check if first URL is the same
videoURL = currThread.downloadList[0][0]
videoURL = serverURLs[thread.threadID] + re.findall("[0-9]/.*", videoURL)[0][1:]
if [videoURL, currThread.downloadList[0][1], currThread.downloadList[0][2]] == thread.downloadList[0]:
currThread.downloadSize = -1
# check if its time to start redoing work
timePassed = datetime.now() - thread.beginTime
if timePassed.seconds >= redoTime:
for currThread in self.threads:
totalDownloaded = 0
totalSize = 0
if currThread.isAlive() and not currThread.downloadSize is not -1:
# check progress to see if thread's work is worth redoing
for index, download in zip(currThread.bufferIndices, currThread.downloadList):
# check full file size
if download[1] is None:
size = manifest.readSize(bitrates[bitrateIndex], index)
else:
size = download[2] - download[1] + 1
totalSize += size
# check downloaded size
downloaded = self.buffer.getFileSize(index, download[1], download[2])
if downloaded == -1:
downloaded = size
totalDownloaded += downloaded
work = totalSize - totalDownloaded
if work > 500:
# make a modified copy of downloads for redoing work
downloadList = []
for download in currThread.downloadList:
videoURL = download[0]
videoURL = serverURLs[thread.threadID] + re.findall("[0-9]/.*", videoURL)[0][1:]
downloadList.append([videoURL, download[1], download[2]])
self.threads[thread.threadID] = Downloader(thread.threadID, downloadList, self.buffer,
currThread.bufferIndices, thread.bandwidth, None, thread.connectionBeginTime, True)
self.threads[thread.threadID].start()
idleThreadIDs.remove(thread.threadID)
while self.threads[thread.threadID].beginTime is None:
pass
break
# start new round
if startNewRound and self.buffer.size - self.buffer.length >= NUM_THREADS:
# bitrate adaptation based on bandwidth
if chunkIndex != 1:
time = 0
min = None
max = None
for currThread in self.threads:
if min == None or currThread.connectionBeginTime < min:
min = currThread.connectionBeginTime
if max == None or currThread.endTime > max:
max = currThread.endTime
deltaT = max - min
time = deltaT.microseconds + deltaT.seconds * 1000000
u = float(CHUNK_DURATION * NUM_THREADS) / time
print time
print u
if u > 1 + epsilon:
if bitrateIndex != len(bitrates) - 1:
bitrateIndex += 1
elif u < 1:
if bitrateIndex != 0:
bitrateIndex -= 1
#threadBandwidth[thread.threadID] += u
# Get URLS and chunk sizes from manifest file based on bitrate and chunk index
chunks = []
totalSize = 0
for i in range(NUM_THREADS):
if chunkIndex == numChunks + 1:
break
URLend, chunkSize = manifest.read(bitrates[bitrateIndex], chunkIndex + i)
URLend = "/" + URLend
self.buffer.setSize(chunkIndex + i, chunkSize)
chunks.append([URLend, 0, chunkSize - 1, chunkIndex + i])
totalSize += chunkSize
# Get threadID's and their threadInfo
threadInfo = []
totalBandwidth = 0
for currThread in self.threads:
totalBandwidth += currThread.bandwidth
threadInfo.append([currThread.threadID, currThread.bandwidth])
# sort threadInfo so that higher bandwidth connections get assigned
# to earlier byte ranges
threadInfo = sorted(threadInfo, key=lambda list: list[1], reverse = True)
for info in threadInfo:
currThread = self.threads[info[0]]
currThread.downloadList = []
currThread.bufferIndices = []
fraction = float(currThread.bandwidth) / totalBandwidth
allocation = int(totalSize * fraction)
print str(info[0]) + " downloading: " + str(allocation) + " " + str(fraction)
for chunk in chunks:
if chunk[1] == chunk[2] + 1:
continue
elif allocation == 0:
break
# due to possible imprecision errors with calculating allocation,
# check to have last thread download remains of last chunk
if info == threadInfo[len(threadInfo) - 1] and chunk == chunks[len(chunks) - 1]:
if chunk[1] == 0:
currThread.downloadList.append((serverURLs[currThread.threadID] + chunk[0],
None, None))
else:
currThread.downloadList.append((serverURLs[currThread.threadID] + chunk[0],
chunk[1], chunk[2]))
allocation = 0
chunk[1] = chunk[2] + 1
currThread.bufferIndices.append(chunk[3])
elif allocation - (chunk[2] - chunk[1] + 1) >= 0:
# assign thread to download rest of chunk
if chunk[1] == 0:
currThread.downloadList.append((serverURLs[currThread.threadID] + chunk[0],
None, None))
else:
currThread.downloadList.append((serverURLs[currThread.threadID] + chunk[0],
chunk[1], chunk[2]))
allocation -= (chunk[2] - chunk[1] + 1)
chunk[1] = chunk[2] + 1
currThread.bufferIndices.append(chunk[3])
else:
# assign thread to download part of chunk
# to finish its allocation
currThread.downloadList.append((serverURLs[currThread.threadID] + chunk[0],
chunk[1], chunk[1] + allocation - 1))
chunk[1] += allocation
allocation = 0
currThread.bufferIndices.append(chunk[3])
# Keep track of bitrate for graphing purposes
bitrateFreq[bitrates[bitrateIndex]] += NUM_THREADS
# all threads have been assigned to download
idleThreadIDs = Set()
startNewRound = False
# Request urls from server
for currThread in self.threads:
ID = currThread.threadID
self.threads[ID] = Downloader(ID, currThread.downloadList, self.buffer, currThread.bufferIndices, currThread.bandwidth)
self.threads[ID].start()
self.buffer.length += NUM_THREADS
chunkIndex += NUM_THREADS
# at every round, plot figure using existing information"
statBitrateFreq = {}
statThreadBandwidth = {}
#print "Chunk Index", chunkIndex
#print bitrateFreq
#print threadBandwidth
for key in bitrateFreq.keys():
value = bitrateFreq[key]
statBitrateFreq[key] = (float(value)/chunkIndex) * 100
#for key in threadBandwidth.keys():
# value = threadBandwidth[key]
# statThreadBandwidth[key] = value/chunkIndex
self.plot(statBitrateFreq, "Pure Byte Range Method")