def __init__(self, threadMgr, cat = None, name = 'agent_thread', mergeOnFound = False, reqChecksum = None): """ Constructor. Creates the uuid. Sets the category of this thread """ threading.Thread.__init__(self) self.__lock = threading.Lock() self.__stop = False self._timeout = configutil.getConfigAsInt('agent_thread_timeout') self._progressTimeout = 0 self.__lastProgress = None self.__timeoutAt = 0 # thread event to mark when the thread has been added to threadMgr self.threadMgrEvent = threading.Event() self.threadMgrEvent.clear() # thread manager self._threadMgr = threadMgr # used by status self.__uuid = str(uuid.uuid4()) self.__cat = cat self.__name = name self.__executionMsec = None self.__mergeOnFound = mergeOnFound self.__reqChecksum = reqChecksum # status self.__status = {'httpStatus': 200, 'progress': 0, 'fprogress': 0.0, 'result': None, 'error': None, 'errorMsg': None, 'executionMsec': None}
def __init__(self, threadMgr, stopOnly=False): """ Constructor """ AgentThread.__init__(self, threadMgr, cat=[ServicesCleanup.CAT], name='cleanup_service') self.__killTimeout = configutil.getConfigAsInt( 'agent_thread_timeout') #kill -9 should take no time self.__stopOnly = stopOnly
def __startDownload(self): """ actual download logic """ try: LOG.info("Starting package download for package %s" % self.__uriDict['package']) # check to see if there's an in progress file, # since PackageMgr guarantees that duplicate threads will not be spawned # for same pkg, assume an existing thread was killed. # attempt to clean up package n move if (os.path.exists(self.__uriDict['inProgressPackagePath'])): LOG.debug('In progress file (%s) already exists. Cleanup and reattempt download' % self.__uriDict['inProgressPackagePath']) if os.path.exists(self.__uriDict['packagePath']): if self.__skipProp or ((os.path.exists(self.__uriDict['propPath']) and PackageUtil.validateProp(self.__uriDict['propPath']) and PackageUtil.validatePackage(self.__uriDict['packagePath'], self.__uriDict['propPath']))): msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict['packagePath'] LOG.info(msg) os.utime(self.__uriDict['packagePath'], None) if os.path.exists(self.__uriDict['propPath']): os.utime(self.__uriDict['propPath'], None) self._updateStatus(progress = 100) # NOTE: this is a normal exit not an error! return else: LOG.warning('The package already exists. However package prop (%s) failed validation.' % self.__uriDict['propPath']) # Delete all traces of package before beginning download LOG.debug('Cleaning up all packages for %s ' % self.__uriDict['packagePath']) PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath'], self.__uriDict['propPath']) AgentThread._updateProgress(self, 0) if self.__skipProp: LOG.info('Skip download of prop file') else: # First, download .prop file LOG.info('Starting download of prop file %s - %s' % (self.__uriDict['propUri'], self.__uriDict['propPath'])) self.__download_prop_file() try: self.__prop = loadPropFile(self.__uriDict['propPath']) except FileNotFoundError: raise AgentException(Errors.DC_MISSING_PROP_FILE, 'Prop file (%s) unable to read or did not parse' % (self.__uriDict['propPath'])) AgentThread._updateProgress(self, 2) self.__setProgressTimeouts() if self.__uriDict['scheme'] == 'http': # try download 3 times, with random sleep attempt = configutil.getConfigAsInt('download_thread_attempt') for _ in range(attempt): try: sotimeout = float(pylons.config['download_thread_sotimeout']) proxies = json.loads(pylons.config['urlgrabber_proxies']) urlgrabber.urlgrab(self.__uriDict['uri'], self.__uriDict['inProgressPackagePath'], # checkfunc = None if self.__skipProp else (PackageUtil.validateDownload, (), {}), progress_obj = DownloadProgress(self), throttle = float(pylons.config['package_throttle']), bandwidth = int(pylons.config['package_bandwidth']), keepalive = 0, timeout = sotimeout, proxies = proxies) break except Exception as exc: msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3)) LOG.warning(msg) if _ == attempt-1: raise exc randsleep = randint(5, 10) time.sleep(randsleep) else: # oops! only http supported now raise AgentException(Errors.DC_UNSUPPORTED_PROTOCOL, 'Only http protocols is supported at the moment') self._checkStop() if self.__skipProp: LOG.info('Skip validating against prop file') else: if (not PackageUtil.validatePackage(self.__uriDict['inProgressPackagePath'], self.__uriDict['propPath'])): raise AgentException(Errors.DC_FAILED_VALIDATE, 'Package ' + self.__uriDict['packagePath'] + ' failed validation') os.utime(self.__uriDict['propPath'], None) utils.rchmod(self.__uriDict['propPath'], "777", 'no') LOG.info('Download complete, rename this file %s' % self.__uriDict['packagePath']) os.rename(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath']) os.utime(self.__uriDict['packagePath'], None) utils.rchmod(self.__uriDict['packagePath'], "777", 'no') LOG.info("Download complete, updating progress to 100") self._updateStatus(progress = 100) except AgentException, exc: self._updateStatus(httpStatus = 500, progress = 0, error = exc.getCode(), errorMsg = exc.getMsg()) msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3)) LOG.error(msg) raise exc
def __init__(self, threadMgr, stopOnly=False): """ Constructor """ AgentThread.__init__(self, threadMgr, cat = [ServicesCleanup.CAT], name = 'cleanup_service') self.__killTimeout = configutil.getConfigAsInt('agent_thread_timeout') #kill -9 should take no time self.__stopOnly = stopOnly