def startdownload(self):
"""
Download using http. The protocol is chosen based on package uri.
Target folder is packageloc relative to repo_root.
"""
try:
utils.checkDiskFull()
reqjson = json.loads(request.body)
package = reqjson['package']
packageloc = str(reqjson['packageloc'])
LOG.info('Request received for StartDownload %s' % packageloc)
appGlobal = config['pylons.app_globals']
LOG.info('Starting a new StartDownload Thread %s' % package)
if type(package) == list:
downloadThread = PackageDownload(appGlobal.threadMgr, package)
else:
cat = PackageUtil.getPackageKey(package)
downloadThread = DownloadThread(appGlobal.threadMgr, package, packageloc, category = [cat])
self.injectJobCtx(downloadThread)
downloadThread.start()
downloadThread.threadMgrEvent.wait()
return statusResult(request, response, downloadThread, controller = self)
except AgentException as excep:
return errorResult(request, response, error = excep.getCode(), errorMsg = excep.getMsg(), controller = self)
except Exception as excp:
errorMsg = 'Exception downloading %s - traceback %s' % (str(excp), traceback.format_exc(2))
return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg = errorMsg, controller = self)
def _downloadPackages(self, packages, isFailedFatal = True):
"""
download all the packages
update progress
check for timeout
@params packages = list of package uri's
@throws AgentException
"""
try:
appGlobal = pylons.config['pylons.app_globals']
# globals all the remaining packages, download the packages
pkgObjs = []
for pkg in packages:
packageKey = PackageUtil.getPackageKey(pkg)
matchDlThreads = appGlobal.threadMgr.getThreadByCat(packageKey)
if matchDlThreads:
# found an inprogress download thread
dlThread = matchDlThreads[0]
else:
dlThread = DownloadThread(self._threadMgr, pkg, category=[packageKey], packageloc=None, parentId = self.getUuid())
contextutils.copyJobContexts(self, dlThread)
dlThread.start()
if (dlThread.getStatus().get('progress') != 100):
pkgObjs.append(dlThread)
self._checkStop()
# check that we have packages to download
if (len(pkgObjs) == 0):
return [] # nothing to download, so no failed packages
# now wait for all the packages to finish
liveThreadCount = len(pkgObjs)
timeoutNotSet = True
create_sleep_time = float(pylons.config['exec_thread_sleep_time']) * liveThreadCount
failed_packages = []
while (liveThreadCount > 0):
self._checkStop()
LOG.info('%s packages still downloading' % liveThreadCount)
time.sleep(create_sleep_time)
# go through all the packages and
# calculated average progress
# check that all packages are alive
totalProgress = 0
liveThreadCount = 0
timeouts = (0.0, 0.0)
for dlThread in pkgObjs:
# we are not done yet
if dlThread.isAlive():
liveThreadCount += 1
# if one package failed, then we have to fail the entire manifest
threadStatus = dlThread.getStatus()
if (not dlThread.isAlive() and threadStatus.get('progress') != 100):
if isFailedFatal:
raise AgentException(Errors.DC_FAILED_DOWNLOAD,
'failed downloading package (%s) - %s'
% (dlThread.getUriDict().get('uri'), threadStatus.get('errorMsg')))
failed_packages.append(dlThread.getUriDict().get('uri'))
progressTimeout = dlThread.getProgressTimeouts()
if (progressTimeout and timeouts):
timeouts = (timeouts[0] + progressTimeout[0], timeouts[1] + progressTimeout[1])
else:
timeouts = None
pkgProgress = threadStatus.get('progress')
totalProgress += pkgProgress
if (timeouts and timeoutNotSet):
# Setting the timeout once as it is absolute time. Doing this after the timeout for each
# package is available
self.extendTimeout(timeouts[0])
timeoutNotSet = False
LOG.debug('Using overall timeout=%s and progress timeout=%s' % (timeouts[0], timeouts[1]))
# calculate the new progress
# I'm allocating 80 of the progress to be the download
newProgress = calcProgress(1, 79, float(totalProgress) / (100 * len(pkgObjs)))
# now update the pgoress
self._updateProgress(newProgress)
return failed_packages
finally:
# now update the pgoress
self._updateProgress(79)
