def __init__(self, threadMgr, cat = None, name = 'agent_thread', mergeOnFound = False, reqChecksum = None):
""" Constructor.
Creates the uuid.
Sets the category of this thread """
threading.Thread.__init__(self)
self.__lock = threading.Lock()
self.__stop = False
self._timeout = configutil.getConfigAsInt('agent_thread_timeout')
self._progressTimeout = 0
self.__lastProgress = None
self.__timeoutAt = 0
# thread event to mark when the thread has been added to threadMgr
self.threadMgrEvent = threading.Event()
self.threadMgrEvent.clear()
# thread manager
self._threadMgr = threadMgr
# used by status
self.__uuid = str(uuid.uuid4())
self.__cat = cat
self.__name = name
self.__executionMsec = None
self.__mergeOnFound = mergeOnFound
self.__reqChecksum = reqChecksum
# status
self.__status = {'httpStatus': 200, 'progress': 0, 'fprogress': 0.0, 'result': None, 'error': None, 'errorMsg': None, 'executionMsec': None}
def __init__(self, threadMgr, stopOnly=False):
""" Constructor """
AgentThread.__init__(self,
threadMgr,
cat=[ServicesCleanup.CAT],
name='cleanup_service')
self.__killTimeout = configutil.getConfigAsInt(
'agent_thread_timeout') #kill -9 should take no time
self.__stopOnly = stopOnly
def __startDownload(self):
""" actual download logic """
try:
LOG.info("Starting package download for package %s" % self.__uriDict['package'])
# check to see if there's an in progress file,
# since PackageMgr guarantees that duplicate threads will not be spawned
# for same pkg, assume an existing thread was killed.
# attempt to clean up package n move
if (os.path.exists(self.__uriDict['inProgressPackagePath'])):
LOG.debug('In progress file (%s) already exists. Cleanup and reattempt download'
% self.__uriDict['inProgressPackagePath'])
if os.path.exists(self.__uriDict['packagePath']):
if self.__skipProp or ((os.path.exists(self.__uriDict['propPath']) and
PackageUtil.validateProp(self.__uriDict['propPath']) and
PackageUtil.validatePackage(self.__uriDict['packagePath'],
self.__uriDict['propPath']))):
msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict['packagePath']
LOG.info(msg)
os.utime(self.__uriDict['packagePath'], None)
if os.path.exists(self.__uriDict['propPath']):
os.utime(self.__uriDict['propPath'], None)
self._updateStatus(progress = 100)
# NOTE: this is a normal exit not an error!
return
else:
LOG.warning('The package already exists. However package prop (%s) failed validation.'
% self.__uriDict['propPath'])
# Delete all traces of package before beginning download
LOG.debug('Cleaning up all packages for %s ' % self.__uriDict['packagePath'])
PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'],
self.__uriDict['packagePath'],
self.__uriDict['propPath'])
AgentThread._updateProgress(self, 0)
if self.__skipProp:
LOG.info('Skip download of prop file')
else:
# First, download .prop file
LOG.info('Starting download of prop file %s - %s' % (self.__uriDict['propUri'], self.__uriDict['propPath']))
self.__download_prop_file()
try:
self.__prop = loadPropFile(self.__uriDict['propPath'])
except FileNotFoundError:
raise AgentException(Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) unable to read or did not parse' % (self.__uriDict['propPath']))
AgentThread._updateProgress(self, 2)
self.__setProgressTimeouts()
if self.__uriDict['scheme'] == 'http':
# try download 3 times, with random sleep
attempt = configutil.getConfigAsInt('download_thread_attempt')
for _ in range(attempt):
try:
sotimeout = float(pylons.config['download_thread_sotimeout'])
proxies = json.loads(pylons.config['urlgrabber_proxies'])
urlgrabber.urlgrab(self.__uriDict['uri'],
self.__uriDict['inProgressPackagePath'],
# checkfunc = None if self.__skipProp else (PackageUtil.validateDownload, (), {}),
progress_obj = DownloadProgress(self),
throttle = float(pylons.config['package_throttle']),
bandwidth = int(pylons.config['package_bandwidth']),
keepalive = 0,
timeout = sotimeout,
proxies = proxies)
break
except Exception as exc:
msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3))
LOG.warning(msg)
if _ == attempt-1:
raise exc
randsleep = randint(5, 10)
time.sleep(randsleep)
else:
# oops! only http supported now
raise AgentException(Errors.DC_UNSUPPORTED_PROTOCOL, 'Only http protocols is supported at the moment')
self._checkStop()
if self.__skipProp:
LOG.info('Skip validating against prop file')
else:
if (not PackageUtil.validatePackage(self.__uriDict['inProgressPackagePath'],
self.__uriDict['propPath'])):
raise AgentException(Errors.DC_FAILED_VALIDATE, 'Package ' +
self.__uriDict['packagePath'] + ' failed validation')
os.utime(self.__uriDict['propPath'], None)
utils.rchmod(self.__uriDict['propPath'], "777", 'no')
LOG.info('Download complete, rename this file %s' % self.__uriDict['packagePath'])
os.rename(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath'])
os.utime(self.__uriDict['packagePath'], None)
utils.rchmod(self.__uriDict['packagePath'], "777", 'no')
LOG.info("Download complete, updating progress to 100")
self._updateStatus(progress = 100)
except AgentException, exc:
self._updateStatus(httpStatus = 500, progress = 0, error = exc.getCode(), errorMsg = exc.getMsg())
msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3))
LOG.error(msg)
raise exc
def __init__(self, threadMgr, stopOnly=False):
""" Constructor """
AgentThread.__init__(self, threadMgr, cat = [ServicesCleanup.CAT], name = 'cleanup_service')
self.__killTimeout = configutil.getConfigAsInt('agent_thread_timeout') #kill -9 should take no time
self.__stopOnly = stopOnly
