def FindLatestManifest(self, train = None):
# Gets <UPDATE_SERVER>/<train>/LATEST
# Returns a manifest, or None.
rv = None
current_version = None
temp_mani = self.SystemManifest()
if temp_mani:
current_version = temp_mani.Sequence()
if train is None:
if temp_mani is None:
# I give up
raise Exceptions.ConfigurationInvalidException
if temp_mani.NewTrain():
# If we're redirected to a new train, use that.
train = temp_mani.NewTrain()
else:
train = temp_mani.Train()
file = self.TryGetNetworkFile("%s/%s/LATEST" % (UPDATE_SERVER, train))
if file is None:
log.debug("Could not get latest manifest file for train %s" % train)
else:
rv = Manifest.Manifest(self)
rv.LoadFile(file)
return rv
def __init__(self,
itemHash,
itemInstanceId,
sockets=None,
displayProperties=None,
inventory=None,
categories=None,
stats=None,
manifestId=None):
self.itemHash = itemHash
self.itemInstanceId = itemInstanceId
self.manifestId = manifestId if manifestId else Manifest.HashtoID(
itemHash)
self.displayProperties = displayProperties
self.inventory = inventory
self.categories = categories
self.stats = stats if isinstance(stats, dict) else dict()
self.sockets = []
if sockets:
for a in sockets:
if isinstance(a, SocketSet):
self.sockets.append(a)
else:
sockets.append(SocketSet(**a))
if not all((self.manifestId, self.displayProperties, self.inventory,
self.categories)):
self.queryManifest()
def SystemManifest(self):
man = Manifest.Manifest(self)
try:
man.LoadPath(self._root + Manifest.SYSTEM_MANIFEST_FILE)
except:
man = None
return man
def run_all_manifests():
"""
Test creation of a KOPS kubeconfig file via Jinja2 template
:return:
"""
print(
"***************************************************************************************************"
)
print("Configure Manifest for Cluster...")
print(
"***************************************************************************************************"
)
Manifest(manifest_template_file, manifest_data_file, new_manifest_file,
state_template_file, state_shell_file, vers_template_file,
vers_shell_file, ssh_template_file, ssh_shell_file,
upd_template_file, upd_shell_file, val_template_file,
val_shell_file, kops_template_file, kops_shell_file,
kops_del_template_file, kops_del_shell_file, chef_manifest_file,
chef_shell_file, chef_nofips_manifest_file,
chef_nofips_shell_file, pre_build_manifest_file,
pre_build_shell_file, ca_cert_file, client_key_file)
print(
"***************************************************************************************************"
)
print("Manifest Configuration Complete")
print(
"***************************************************************************************************"
)
return
def __init__(self, plugHash, manifestId=None, displayProperties=None):
self.plugHash = plugHash
self.manifestId = manifestId
self.displayProperties = displayProperties
if not all((self.manifestId, self.displayProperties)):
self.manifestId = Manifest.HashtoID(plugHash)
self.queryManifest()
def FindLatestManifest(self, train = None, require_signature = False):
# Gets <UPDATE_SERVER>/<train>/LATEST
# Returns a manifest, or None.
rv = None
temp_mani = self.SystemManifest()
if train is None:
if temp_mani is None:
# I give up
raise Exceptions.ConfigurationInvalidException
if temp_mani.NewTrain():
# If we're redirected to a new train, use that.
train = temp_mani.NewTrain()
else:
train = temp_mani.Train()
file = self.TryGetNetworkFile(url = "%s/%s/LATEST" % (self.UpdateServerMaster(), train),
reason = "GetLatestManifest",
)
if file is None:
log.debug("Could not get latest manifest file for train %s" % train)
else:
rv = Manifest.Manifest(self, require_signature = require_signature)
rv.LoadFile(file)
return rv
def queryManifest(self):
wcp = Manifest.getDatabaseConnection()
definition = wcp.selectQuery(self.manifestId,
'DestinyInventoryItemDefinition')
self.displayProperties = definition['displayProperties']
self.inventory = definition['inventory']
self.categories = definition['itemCategoryHashes'] if 'itemCategoryHashes' in definition else []
def SystemManifest(self):
if self._manifest is None:
self._manifest = Manifest.Manifest(configuration=self)
try:
self._manifest.LoadPath(self._root +
Manifest.SYSTEM_MANIFEST_FILE)
except:
self._manifest = None
return self._manifest
def getCharacterInfo(session, membershiptype, membershipid, characters):
WCP = Manifest.getDatabaseConnection()
charIDs = characters
characterinfo = CharacterList()
for char in charIDs:
a = Character(char)
result = getCharacter(session, membershiptype, membershipid, char,
'200')
a.emblemBackgroundPath = result['character']['data'][
'emblemBackgroundPath']
a.light = result['character']['data']['light']
classid = Manifest.HashtoID(result['character']['data']['classHash'])
a.classname = WCP.selectQuery(
classid, 'DestinyClassDefinition')['displayProperties']['name']
raceid = Manifest.HashtoID(result['character']['data']['raceHash'])
a.racename = WCP.selectQuery(
raceid, 'DestinyRaceDefinition')['displayProperties']['name']
genderid = Manifest.HashtoID(result['character']['data']['genderHash'])
a.gendername = WCP.selectQuery(
genderid, 'DestinyGenderDefinition')['displayProperties']['name']
characterinfo.chars.append(a)
return characterinfo
def FindLatestManifest(self, train=None):
# Finds the latest (largest sequence number)
# manifest for a given train, iterating through
# the search locations.
# Returns a manifest, or None.
rv = None
if train is None:
temp_mani = self.SystemManifest()
if temp_mani is None:
# I give up
raise ConfigurationInvalidException
train = temp_mani.Train()
for file in self.SearchForFile(train + "/LATEST"):
temp_mani = Manifest.Manifest(self)
temp_mani.LoadFile(file)
if rv is None or temp_mani.Sequence() > rv.Sequence():
rv = temp_mani
return rv
def streamMethod2(self):
chunkIndex = 1
redoTime = 0
manifest = Manifest("BunnyManifest_1.xml")
bitrates = manifest.listOfBitrates()
# Compute switch-up epsilon
epsilon = self.computeEpsilon(bitrates)
#start with lowest bitrate
bitrateIndex = 0
numChunks = 99 #manifest.numberOfChunks(bitrates[bitrateIndex])
idleThreadIDs = Set()
startNewRound = False
#threadBandwidth = {}
for thread in self.threads:
if thread.successful:
idleThreadIDs.add(thread.threadID)
#threadBandwidth[thread.threadID] = 0
if len(idleThreadIDs) == NUM_THREADS:
startNewRound = True
finished = False
bitrateFreq = {}
for bitrate in bitrates:
bitrateFreq[bitrate] = 0
# Start Player
#p = Player(self.buffer)
#p.start()
while not finished:
for thread in self.threads:
if not thread.isAlive():
# Check if we have gotten all the chunks
# If so, we are done
if chunkIndex == numChunks + 1:
finished = True
break
# if threads finish after round of downloads started, record their IDs
if not startNewRound and thread.successful:
idleThreadIDs.add(thread.threadID)
# if all threads finish, start new round of downloads
if len(idleThreadIDs) == NUM_THREADS:
startNewRound = True
else:
# if finished downloading file first, stop duplicate downloads
for currThread in self.threads:
if currThread.isAlive():
# just check if first URL is the same
videoURL = currThread.downloadList[0][0]
videoURL = serverURLs[thread.threadID] + re.findall("[0-9]/.*", videoURL)[0][1:]
if [videoURL, currThread.downloadList[0][1], currThread.downloadList[0][2]] == thread.downloadList[0]:
currThread.downloadSize = -1
# check if its time to start redoing work
timePassed = datetime.now() - thread.beginTime
if timePassed.seconds >= redoTime:
for currThread in self.threads:
totalDownloaded = 0
totalSize = 0
if currThread.isAlive() and not currThread.downloadSize is not -1:
# check progress to see if thread's work is worth redoing
for index, download in zip(currThread.bufferIndices, currThread.downloadList):
# check full file size
if download[1] is None:
size = manifest.readSize(bitrates[bitrateIndex], index)
else:
size = download[2] - download[1] + 1
totalSize += size
# check downloaded size
downloaded = self.buffer.getFileSize(index, download[1], download[2])
if downloaded == -1:
downloaded = size
totalDownloaded += downloaded
work = totalSize - totalDownloaded
if work > 500:
# make a modified copy of downloads for redoing work
downloadList = []
for download in currThread.downloadList:
videoURL = download[0]
videoURL = serverURLs[thread.threadID] + re.findall("[0-9]/.*", videoURL)[0][1:]
downloadList.append([videoURL, download[1], download[2]])
self.threads[thread.threadID] = Downloader(thread.threadID, downloadList, self.buffer,
currThread.bufferIndices, thread.bandwidth, None, thread.connectionBeginTime, True)
self.threads[thread.threadID].start()
idleThreadIDs.remove(thread.threadID)
while self.threads[thread.threadID].beginTime is None:
pass
break
# start new round
if startNewRound and self.buffer.size - self.buffer.length >= NUM_THREADS:
# bitrate adaptation based on bandwidth
if chunkIndex != 1:
time = 0
min = None
max = None
for currThread in self.threads:
if min == None or currThread.connectionBeginTime < min:
min = currThread.connectionBeginTime
if max == None or currThread.endTime > max:
max = currThread.endTime
deltaT = max - min
time = deltaT.microseconds + deltaT.seconds * 1000000
u = float(CHUNK_DURATION * NUM_THREADS) / time
print time
print u
if u > 1 + epsilon:
if bitrateIndex != len(bitrates) - 1:
bitrateIndex += 1
elif u < 1:
if bitrateIndex != 0:
bitrateIndex -= 1
#threadBandwidth[thread.threadID] += u
# Get URLS and chunk sizes from manifest file based on bitrate and chunk index
chunks = []
totalSize = 0
for i in range(NUM_THREADS):
if chunkIndex == numChunks + 1:
break
URLend, chunkSize = manifest.read(bitrates[bitrateIndex], chunkIndex + i)
URLend = "/" + URLend
self.buffer.setSize(chunkIndex + i, chunkSize)
chunks.append([URLend, 0, chunkSize - 1, chunkIndex + i])
totalSize += chunkSize
# Get threadID's and their threadInfo
threadInfo = []
totalBandwidth = 0
for currThread in self.threads:
totalBandwidth += currThread.bandwidth
threadInfo.append([currThread.threadID, currThread.bandwidth])
# sort threadInfo so that higher bandwidth connections get assigned
# to earlier byte ranges
threadInfo = sorted(threadInfo, key=lambda list: list[1], reverse = True)
for info in threadInfo:
currThread = self.threads[info[0]]
currThread.downloadList = []
currThread.bufferIndices = []
fraction = float(currThread.bandwidth) / totalBandwidth
allocation = int(totalSize * fraction)
print str(info[0]) + " downloading: " + str(allocation) + " " + str(fraction)
for chunk in chunks:
if chunk[1] == chunk[2] + 1:
continue
elif allocation == 0:
break
# due to possible imprecision errors with calculating allocation,
# check to have last thread download remains of last chunk
if info == threadInfo[len(threadInfo) - 1] and chunk == chunks[len(chunks) - 1]:
if chunk[1] == 0:
currThread.downloadList.append((serverURLs[currThread.threadID] + chunk[0],
None, None))
else:
currThread.downloadList.append((serverURLs[currThread.threadID] + chunk[0],
chunk[1], chunk[2]))
allocation = 0
chunk[1] = chunk[2] + 1
currThread.bufferIndices.append(chunk[3])
elif allocation - (chunk[2] - chunk[1] + 1) >= 0:
# assign thread to download rest of chunk
if chunk[1] == 0:
currThread.downloadList.append((serverURLs[currThread.threadID] + chunk[0],
None, None))
else:
currThread.downloadList.append((serverURLs[currThread.threadID] + chunk[0],
chunk[1], chunk[2]))
allocation -= (chunk[2] - chunk[1] + 1)
chunk[1] = chunk[2] + 1
currThread.bufferIndices.append(chunk[3])
else:
# assign thread to download part of chunk
# to finish its allocation
currThread.downloadList.append((serverURLs[currThread.threadID] + chunk[0],
chunk[1], chunk[1] + allocation - 1))
chunk[1] += allocation
allocation = 0
currThread.bufferIndices.append(chunk[3])
# Keep track of bitrate for graphing purposes
bitrateFreq[bitrates[bitrateIndex]] += NUM_THREADS
# all threads have been assigned to download
idleThreadIDs = Set()
startNewRound = False
# Request urls from server
for currThread in self.threads:
ID = currThread.threadID
self.threads[ID] = Downloader(ID, currThread.downloadList, self.buffer, currThread.bufferIndices, currThread.bandwidth)
self.threads[ID].start()
self.buffer.length += NUM_THREADS
chunkIndex += NUM_THREADS
# at every round, plot figure using existing information"
statBitrateFreq = {}
statThreadBandwidth = {}
#print "Chunk Index", chunkIndex
#print bitrateFreq
#print threadBandwidth
for key in bitrateFreq.keys():
value = bitrateFreq[key]
statBitrateFreq[key] = (float(value)/chunkIndex) * 100
#for key in threadBandwidth.keys():
# value = threadBandwidth[key]
# statThreadBandwidth[key] = value/chunkIndex
self.plot(statBitrateFreq, "Pure Byte Range Method")
def __init__(self, label, rerun, cameraInfo, dataDir, **kwargs):
"""
@param label A label to refer to the data
@param rerun The rerun to retrieve
@param cameraInfo A cameraInfo object describing the camera for these data
@param dataDir The full path to the directory containing the data registry file.
@param haveManifest verify files in dataDir are present according to manifest
@param verifyChecksum verify files in dataDir have correct checksum as listed in manifest
"""
QaData.__init__(self, label, rerun, cameraInfo)
self.rerun = rerun
self.dataDir = dataDir
###############################################
# handle keyword args
###############################################
self.kwargs = kwargs
self.dataId = self.kwargs.get('dataId', {})
self.haveManifest = self.kwargs.get('haveManifest', False)
self.verifyChecksum = self.kwargs.get('verifyChecksum', False)
self.shapeAlg = self.kwargs.get('shapeAlg', 'HSM_REGAUSS')
knownAlgs = ["HSM_REGAUSS", "HSM_BJ", "HSM_LINEAR", "HSM_SHAPELET", "HSM_KSB"]
if not self.shapeAlg in set(knownAlgs):
knownStr = "\n".join(knownAlgs)
raise Exception("Unknown shape algorithm: %s. Please choose: \n%s\n" % (self.shapeAlg, knownStr))
###############################################
# check the manifest, if requested
# haveManifest = True is a bit slowish
# verifyChecksum = True is quite slow
manifest.verifyManifest(self.dataDir, verifyExists=self.haveManifest,
verifyChecksum=self.verifyChecksum)
# This (dataId fetching) needs a better design, but will require butler/mapper change, I think.
#
# these obscure things refer to the names assigned to levels in the data hierarchy
# eg. for lsstSim: dataInfo = [['visit',1], ['snap', 0], ['raft',0], ['sensor',0]]
# a level is considered a discriminator if it represents different pictures of the same thing
# ... so the same object may appear in multiple 'visits', but not on multiple 'sensors'
# dataInfo is passed in from the derived class as it's specific to each mapper
dataIdRegexDict = {}
for array in self.dataInfo:
dataIdName, dataIdDiscrim = array
# if the user requested eg. visit=1234.*
# pull that out of kwargs and put it in dataIdRegexDict
if self.dataId.has_key(dataIdName):
dataIdRegexDict[dataIdName] = self.dataId[dataIdName]
#######################################
# get butler
self.outMapper = self.cameraInfo.getMapper(self.dataDir, rerun=self.rerun)
self.outButler = dafPersist.ButlerFactory(mapper=self.outMapper).create()
####################################################
# make a list of the frames we're asked to care about
# get all the available raw inputs
self.availableDataTuples = self.outButler.queryMetadata(cameraInfo.rawName, self.dataIdNames,
format=self.dataIdNames)
# availableDataTuples may be a *very* *large* list. Be sure to call reduceAvailableDataTupleList
self.dataTuples = self.availableDataTuples
if url:
file = self.TryGetNetworkFile(
url = url,
handler = handler,
)
if file:
return file.read()
return None
if __name__ == "__main__":
conf = Configuration()
pkg = Package.Package("freenas", "1.0", "abcd")
pkg.AddUpdate("0.9", "1234")
manifest = Manifest.Manifest(conf)
manifest.SetSequence(100)
manifest.SetTrain("FreeNAS-ALPHA")
manifest.SetVersion("FreeNAS-9.3-ALPHA")
manifest.AddPackage(pkg)
manifest.SetSignature()
if manifest.Validate() != True:
print "Validation failed"
print manifest.String()
manifest.StorePath("manifest")
new_manifest = Manifest.Manifest(conf)
new_manifest.LoadPath("manifest")
if new_manifest.Validate() == True:
print "Re-loaded manifest validated"
from Manifest import *
# pre-write all manifest files (right now just bunny)
bunnyManifest = "Data/BunnyManifest.xml"
#BASE_URL = "http://10.1.3.3:9000/"
f = open(bunnyManifest, "w+")
x = Manifest(bunnyManifest)
x.write("bunny")
f.close()
def queryManifest(self):
wcp = Manifest.getDatabaseConnection()
definition = wcp.selectQuery(self.manifestId,
'DestinyInventoryItemDefinition')
self.displayProperties = definition['displayProperties']
def streamMethod3(self):
chunkIndex = 1
manifest = Manifest("BunnyManifest_1.xml")
bitrates = manifest.listOfBitrates()
# Compute switch-up epsilon
epsilon = self.computeEpsilon(bitrates)
#start with lowest bitrate
bitrateIndex = 0
numChunks = 99 #manifest.numberOfChunks(bitrates[bitrateIndex])
idleThreadIDs = Set()
startNewRound = False
#threadBandwidth = {}
for thread in self.threads:
if thread.successful:
idleThreadIDs.add(thread.threadID)
#threadBandwidth[thread.threadID] = 0
if len(idleThreadIDs) == NUM_THREADS:
startNewRound = True
finished = False
bitrateFreq = {}
for bitrate in bitrates:
bitrateFreq[bitrate] = 0
# Start Player
p = Player(self.buffer)
p.start()
while not finished:
for thread in self.threads:
if not thread.isAlive():
# Check if we have gotten all the chunks
# If so, we are done
if chunkIndex == numChunks + 1:
finished = True
break
# if thread finishes downloading, look for more work
if not startNewRound and thread.successful:
minProgress = None
work = None
videoURL = None
worstThread = None
canHelpNotHelping = False
for currThread in self.threads:
if (currThread.downloadList[0][1] is 0 or currThread.downloadList[0][1] is None) and currThread.isAlive():
canHelpNotHelping = True
break
for currThread in self.threads:
if currThread.isAlive() and (not canHelpNotHelping or (canHelpNotHelping and (currThread.downloadList[0][1] is 0 or currThread.downloadList[0][1] is None))):
size = self.buffer.getFileSize(currThread.bufferIndices[0], currThread.downloadList[0][1], currThread.downloadList[0][2])
# If -1 returned, don't want to help out; just skip
if size == -1:
continue
progress = float(size) / currThread.downloadSize
if minProgress is None or progress < minProgress:
minProgress = progress
work = (currThread.downloadSize - size) / 2
videoURL = currThread.downloadList[0][0]
worstThread = currThread
# if all threads finish or further work is unnecessary, start new round of downloads
if minProgress is None or work < 15000:
idleThreadIDs.add(thread.threadID)
if len(idleThreadIDs) == NUM_THREADS:
startNewRound = True
else:
#split work
ID = thread.threadID
print "splitting: " + str(ID)
# range request for new thread
videoURL = serverURLs[thread.threadID] + re.findall("[0-9]/.*", videoURL)[0][1:]
size = self.buffer.getFileSize(worstThread.bufferIndices[0], worstThread.downloadList[0][1], worstThread.downloadList[0][2])
# calculate new thread's range
if worstThread.downloadList[0][1] is None:
endByte = worstThread.downloadSize - 1
beginByte = size + ((endByte - size) / 2)
#print "None Case"
#print "(begin, end)", beginByte, endByte
else:
endByte = worstThread.downloadList[0][2]
beginByte = worstThread.downloadList[0][1] + size + ((endByte - (worstThread.downloadList[0][1] + size)) / 2)
#print "Other Case"
#print "(begin, end)", beginByte, endByte
downloadSize = endByte - beginByte + 1
# reduce byte range of helped thread
worstThread.downloadSize -= downloadSize
oldBegin = worstThread.downloadList[0][1]
oldEnd = worstThread.downloadList[0][2]
if oldBegin == None:
newBegin = 0
else:
newBegin = oldBegin
newEnd = beginByte - 1
worstThread.downloadList[0][1] = newBegin
worstThread.downloadList[0][2] = newEnd
index = worstThread.bufferIndices[0]
self.buffer.rewriteBeginAndEnd(index, self.buffer.read(index, oldBegin, oldEnd), oldBegin, oldEnd, newBegin, newEnd)
# notify helped thread that its byte range is reduced
worstThread.reduced = True
if ID in idleThreadIDs:
idleThreadIDs.remove(ID)
self.threads[ID] = Downloader(ID, [[videoURL, beginByte, endByte]], self.buffer, [index], thread.bandwidth, downloadSize, thread.connectionBeginTime)
self.threads[ID].start()
while self.buffer.read(index, beginByte, endByte) is "":
pass
# start new round
if startNewRound and self.buffer.size - self.buffer.length >= NUM_THREADS:
# bitrate adaptation based on bandwidth
# do not do for first round
if chunkIndex != 1:
time = 0
min = None
max = None
for currThread in self.threads:
if min == None or currThread.connectionBeginTime < min:
min = currThread.connectionBeginTime
if max == None or currThread.endTime > max:
max = currThread.endTime
deltaT = max - min
time = deltaT.microseconds + deltaT.seconds * 1000000
u = float(CHUNK_DURATION * NUM_THREADS) / time
print time
print u
if u > 1 + epsilon:
if bitrateIndex != len(bitrates) - 1:
bitrateIndex += 1
elif u < 1:
if bitrateIndex != 0:
bitrateIndex -= 1
#threadBandwidth[thread.threadID] += u
# threads have been assigned to download
idleThreadIDs = Set()
# starting of new round finishes when all threads are downloading
startNewRound = False
# Get threadID's and their threadInfo
threadInfo = []
for currThread in self.threads:
threadInfo.append([currThread.threadID, currThread.bandwidth])
# sort threadInfo so that higher bandwidth connections get assigned
# to earlier byte ranges
threadInfo = sorted(threadInfo, key=lambda list: list[1], reverse = True)
for info in threadInfo:
if chunkIndex == numChunks + 1:
break
ID = info[0]
currThread = self.threads[ID]
# Get url from manifest file based on bitrate and chunk index
URLend, chunkSize = manifest.read(bitrates[bitrateIndex], chunkIndex)
videoURL = serverURLs[ID] + "/" + URLend
self.threads[ID] = Downloader(ID, [[videoURL, None, None]], self.buffer, [chunkIndex], currThread.bandwidth, chunkSize)
self.threads[ID].start()
self.buffer.setSize(chunkIndex, chunkSize)
chunkIndex += 1
bitrateFreq[bitrates[bitrateIndex]] += 1
self.buffer.length += 1
# at every round, plot figure using existing information
statBitrateFreq = {}
statThreadBandwidth = {}
#print "Chunk Index", chunkIndex
#print bitrateFreq
#print threadBandwidth
for key in bitrateFreq.keys():
value = bitrateFreq[key]
statBitrateFreq[key] = (float(value)/chunkIndex) * 100
#for key in threadBandwidth.keys():
# value = threadBandwidth[key]
# statThreadBandwidth[key] = value/chunkIndex
self.plot(statBitrateFreq, "Our Method")
def streamMethod1(self):
chunkIndex = 1
manifest = Manifest("BunnyManifest_1.xml")
bitrates = manifest.listOfBitrates()
# Compute switch-up epsilon
epsilon = self.computeEpsilon(bitrates)
#start with lowest bitrate
bitrateIndex = 0
numChunks = 99 #manifest.numberOfChunks(bitrates[bitrateIndex])
idleThreadIDs = Set()
startNewRound = False
#threadBandwidth = {}
for thread in self.threads:
if thread.successful:
idleThreadIDs.add(thread.threadID)
#threadBandwidth[thread.threadID] = 0
if len(idleThreadIDs) == NUM_THREADS:
startNewRound = True
finished = False
# Used for recording bitrate frequences for graphing
bitrateFreq = {}
for bitrate in bitrates:
bitrateFreq[bitrate] = 0
# Start Player
p = Player(self.buffer)
p.start()
while not finished:
for thread in self.threads:
if not thread.isAlive():
# Check if we have gotten all the chunks
# If so, we are done
if chunkIndex == numChunks + 1:
finished = True
break
# if threads finish after round of downloads started, record their IDs
if not startNewRound and thread.successful:
idleThreadIDs.add(thread.threadID)
# if all threads finish, start new round of downloads
if len(idleThreadIDs) == NUM_THREADS:
startNewRound = True
# start new round
if startNewRound and self.buffer.size - self.buffer.length >= NUM_THREADS:
# bitrate adaptation based on bandwidth
# do not do for first round
if chunkIndex != 1:
time = 0
min = None
max = None
for currThread in self.threads:
if min == None or currThread.beginTime < min:
min = currThread.beginTime
if max == None or currThread.endTime > max:
max = currThread.endTime
deltaT = max - min
time = deltaT.microseconds + deltaT.seconds * 1000000
u = float(CHUNK_DURATION * NUM_THREADS) / time
print time
print u
if u > 1 + epsilon:
if bitrateIndex != len(bitrates) - 1:
bitrateIndex += 1
elif u < 1:
if bitrateIndex != 0:
bitrateIndex -= 1
#threadBandwidth[thread.threadID] += u
# threads have been assigned to download
idleThreadIDs = Set()
# starting of new round finishes when all threads are downloading
startNewRound = False
# Request urls from server
for currThread in self.threads:
ID = currThread.threadID
# Get url from manifest file based on bitrate and chunk index
videoURL, chunkSize = manifest.read(bitrates[bitrateIndex], chunkIndex)
videoURL = serverURLs[ID] + "/" + videoURL
bitrateFreq[bitrates[bitrateIndex]] += 1
self.buffer.setSize(chunkIndex, chunkSize)
self.threads[ID] = Downloader(ID, [[videoURL, None, None]], self.buffer, [chunkIndex])
self.threads[ID].start()
chunkIndex += 1
self.buffer.length += 1
# at every round, plot figure using existing information
statBitrateFreq = {}
statThreadBandwidth = {}
# Convert to percentage and graph
for key in bitrateFreq.keys():
value = bitrateFreq[key]
statBitrateFreq[key] = (float(value)/chunkIndex) * 100
self.plot(statBitrateFreq, "Naive Parallel")
def __init__(self, label, rerun, cameraInfo, dataDir, **kwargs):
"""
@param label A label to refer to the data
@param rerun The rerun to retrieve
@param cameraInfo A cameraInfo object describing the camera for these data
@param dataDir The full path to the directory containing the data registry file.
@param haveManifest verify files in dataDir are present according to manifest
@param verifyChecksum verify files in dataDir have correct checksum as listed in manifest
"""
QaData.__init__(self, label, rerun, cameraInfo)
self.rerun = rerun
self.dataDir = dataDir
###############################################
# handle keyword args
###############################################
self.kwargs = kwargs
self.dataId = self.kwargs.get('dataId', {})
self.haveManifest = self.kwargs.get('haveManifest', False)
self.verifyChecksum = self.kwargs.get('verifyChecksum', False)
self.shapeAlg = self.kwargs.get('shapeAlg', 'HSM_REGAUSS')
knownAlgs = ["HSM_REGAUSS", "HSM_BJ", "HSM_LINEAR", "HSM_SHAPELET", "HSM_KSB"]
if not self.shapeAlg in set(knownAlgs):
knownStr = "\n".join(knownAlgs)
raise Exception("Unknown shape algorithm: %s. Please choose: \n%s\n" % (self.shapeAlg, knownStr))
###############################################
# check the manifest, if requested
# haveManifest = True is a bit slowish
# verifyChecksum = True is quite slow
manifest.verifyManifest(self.dataDir, verifyExists=self.haveManifest,
verifyChecksum=self.verifyChecksum)
# This (dataId fetching) needs a better design, but will require butler/mapper change, I think.
#
# these obscure things refer to the names assigned to levels in the data hierarchy
# eg. for lsstSim: dataInfo = [['visit',1], ['snap', 0], ['raft',0], ['sensor',0]]
# a level is considered a discriminator if it represents different pictures of the same thing
# ... so the same object may appear in multiple 'visits', but not on multiple 'sensors'
# dataInfo is passed in from the derived class as it's specific to each mapper
dataIdRegexDict = {}
for array in self.dataInfo:
dataIdName, dataIdDiscrim = array
# if the user requested eg. visit=1234.*
# pull that out of kwargs and put it in dataIdRegexDict
if self.dataId.has_key(dataIdName):
dataIdRegexDict[dataIdName] = self.dataId[dataIdName]
#######################################
# get butler
self.outMapper = self.cameraInfo.getMapper(self.dataDir, rerun=self.rerun)
self.outButler = dafPersist.ButlerFactory(mapper=self.outMapper).create()
####################################################
# make a list of the frames we're asked to care about
# get all the available raw inputs
self.availableDataTuples = self.outButler.queryMetadata(cameraInfo.rawName, self.dataIdNames,
format=self.dataIdNames)
# availableDataTuples may be a *very* *large* list. Be sure to call reduceAvailableDataTupleList
self.dataTuples = self.availableDataTuples
