def __init__(self, threadMgr, uuid): """ Constructor """ AgentThread.__init__(self, threadMgr, cat=[uuid], name='cancel_agentthread') self.__uuid = uuid
def __init__(self, threadMgr, service, manifest, name = 'agent_thread', parentId = None): """ Constructor """ AgentThread.__init__(self, threadMgr, cat = [manifestutil.serviceCat(service)], name = name, parentId = parentId) self.__manifest = manifest self.__service = service contextutils.injectcontext(self, {'service':service}) self._LOG = manifestutil.getServiceLogger(self, logging.getLogger(__name__))
def __init__(self, threadMgr, packageUri, packageloc, path=None, category=None, skipProp=True): # cat = 'DC_DowloadThread' + str(datetime.now()) AgentThread.__init__(self, threadMgr, cat=category, name='download_thread') self.__path = pylons.config['repo_root'] if path is not None: self.__path = path # check to see if the package path exists if (not os.path.isdir(self.__path)): msg = 'Package path(%s) does not exist' % self.__path LOG.error(msg) raise AgentException(Errors.PACKAGE_PATH_ERROR, msg) self.__uriDict = PackageUtil.parseUri(packageUri, self.__path, packageloc) self.__prop = {} self.__error = None self.__progress = 0.0 self.__timeouts = None self.__skipProp = skipProp
def __init__(self, threadMgr, service, manifest): """ Constructor """ from agent.controllers.service import ServiceController AgentThread.__init__(self, threadMgr, cat = [ServiceController.serviceCat(service)], name = 'delete_manifest') self.service = service self.manifest = manifest
def __init__(self, threadMgr, wisbVersion, wisbSource = None): """ Constructor """ AgentThread.__init__(self, threadMgr, cat = [AgentUpdate.CAT], name = AgentUpdate.THREAD_NAME) self.__wisbVersion = wisbVersion self.__wisbSource = wisbSource self.__packages = [] self.__service = 'agent' self.__manifest = ('agent_selfupdate-%s' % self.__wisbVersion)
def __init__(self, threadMgr, service, path): """ Constructor """ AgentThread.__init__(self, threadMgr, cat=[manifestutil.serviceCat(service)], name='delete_service') self.__path = path self.service = service
def __init__(self, threadMgr, module): """ Constructor """ AgentThread.__init__(self, threadMgr, cat=[manifestutil.serviceCat('agent')], name='delete_module') self.__module = module self.__service = 'agent'
def __init__(self, threadMgr, service, manifest, name = 'agent_thread'): """ Constructor """ AgentThread.__init__(self, threadMgr, cat = [manifestutil.serviceCat(service)], name = name) self.__manifest = manifest self.__service = service injectcontext(self, {'service':service}) # maintain a stack that has all the recovery script paths # this stack is protected self._stack = []
def __init__(self, threadMgr, stopOnly=False): """ Constructor """ AgentThread.__init__(self, threadMgr, cat=[ServicesCleanup.CAT], name='cleanup_service') self.__killTimeout = configutil.getConfigAsInt( 'agent_thread_timeout') #kill -9 should take no time self.__stopOnly = stopOnly
def __init__(self, threadMgr, service, manifest): """ Constructor """ from agent.controllers.service import ServiceController AgentThread.__init__(self, threadMgr, cat=[ServiceController.serviceCat(service)], name='delete_manifest') self.service = service self.manifest = manifest
def _updateHttpProgress(self, amount_read): """ custom progress computation """ if not self.__skipProp: progress = 2 + ((float(amount_read) / float(self.__prop['size'])) * (97)) AgentThread._updateProgress(self, progress) else: progress = min(self.getProgress(), 97) + 1 AgentThread._updateProgress(self, progress) self._checkStop()
def __init__(self, threadMgr, service, manifest, name='agent_thread'): """ Constructor """ AgentThread.__init__(self, threadMgr, cat=[manifestutil.serviceCat(service)], name=name) self.__manifest = manifest self.__service = service injectcontext(self, {'service': service}) # maintain a stack that has all the recovery script paths # this stack is protected self._stack = []
def __init__(self, threadMgr, wisbVersion, wisbSource=None, skipProp=True): """ Constructor """ AgentThread.__init__(self, threadMgr, cat=[AgentUpdate.CAT], name=AgentUpdate.THREAD_NAME) self.__wisbVersion = wisbVersion self.__wisbSource = wisbSource self.__packages = [] self.__service = 'agent' self.__manifest = ('agent_selfupdate-%s' % self.__wisbVersion) self.__skipProp = skipProp
def _runExecThreads(self, execThreads, minProgress, maxProgress): """ run exec threads """ if (len(execThreads) == 0): self._updateProgress(maxProgress) return # start with min progress self._updateProgress(minProgress) # now wait for the threads to complete and update progress while (True): self._checkStop() progress = 0 running = False for execThread in execThreads: self.extendTimeout(execThread.getTimeout()) status = AgentThread._runExeThread(self, execThread) progress += status['progress'] if (not running): break # update the progress percent = progress / len(execThreads) self._updateProgress(calcProgress(minProgress, maxProgress, float(percent) / 100)) time.sleep(float(pylons.config['exec_thread_sleep_time'])) self._updateProgress(maxProgress)
def __init__(self, threadMgr, packageUri, packageloc, path = None, category = None, skipProp = True): # cat = 'DC_DowloadThread' + str(datetime.now()) AgentThread.__init__(self, threadMgr, cat = category, name = 'download_thread') self.__path = pylons.config['repo_root'] if path is not None: self.__path = path # check to see if the package path exists if (not os.path.isdir(self.__path)): msg = 'Package path(%s) does not exist' % self.__path LOG.error(msg) raise AgentException(Errors.PACKAGE_PATH_ERROR, msg) self.__uriDict = PackageUtil.parseUri(packageUri, self.__path, packageloc) self.__prop = {} self.__error = None self.__progress = 0.0 self.__timeouts = None self.__skipProp = skipProp
def __init__(self, threadMgr, packageUri, packageloc, path = None, category = None, parentId = None): AgentThread.__init__(self, threadMgr, cat = category, name = 'download_thread', parentId = parentId) self._mergeOnFound = True self.__path = pylons.config['repo_root'] if path is not None: self.__path = path # check to see if the package path exists if (not os.path.isdir(self.__path)): msg = 'Package path(%s) does not exist' % self.__path LOG.error(msg) raise AgentException(Errors.PACKAGE_PATH_ERROR, msg) self.__uriDict = PackageUtil.parseUri(packageUri, self.__path, packageloc) self.__prop = {} self.__error = None self.__progress = 0.0 self.__timeouts = None self.__skipProp = configutil.getConfigAsBool('download_skip_prop')
def __init__(self, threadMgr, cmd, cat = None, parentId = None): """ Constructor. @param threadMgr - the thread Mgr object @param cmd - cmd list @param timeout - amount of time we give to this thread """ AgentThread.__init__(self, threadMgr, cat, name = 'exec_thread', parentId = parentId) self.__cmd = cmd # default timeouts configuration self._timeout = int(config['exec_thread_timeout']) self._progressTimeout = int(config['exec_thread_progress_timeout']) self.__service = None self.__cmdProcess = None self.__response = {} self.__partial_response = None self.__partial = '' self.__outputlines = [] self.__injectctx = True self.__logLevel = 'info' self.__LOG = None
def __init__(self, threadMgr, cmd, cat = None): """ Constructor. @param threadMgr - the thread Mgr object @param cmd - cmd list @param timeout - amount of time we give to this thread """ if threadMgr is None: threadMgr = NullThreadMgr() AgentThread.__init__(self, threadMgr, cat, name = 'exec_thread') self.__cmd = cmd # default timeouts configuration self._timeout = int(config['exec_thread_timeout']) self._progressTimeout = int(config['exec_thread_progress_timeout']) self.__cmdProcess = None self.__response = {} self.__partial_response = None self.__partial = '' self.__logLevel = 'info' service = 'agent' self.__LOG = manifestutil.getServiceLogger(self, logging.getLogger(__name__), service)
def _runExecThreads(self, execThreads, minProgress, maxProgress): """ run exec threads """ if (len(execThreads) == 0): self._updateProgress(maxProgress) return # start with min progress self._updateProgress(minProgress) # now wait for the threads to complete and update progress firstTime = True while (True): self._checkStop() progress = 0 running = False for (execThread, dummy) in execThreads: #first time, push recovery stack if firstTime: self._pushRecStack(execThread.getCmd()) if dummy: continue self.extendTimeout(execThread.getTimeout()) status = AgentThread._runExeThread(self, (execThread, dummy)) progress += status['progress'] firstTime = False if (not running): break # update the progress percent = progress / len(execThreads) self._updateProgress( calcProgress(minProgress, maxProgress, float(percent) / 100)) time.sleep(float(pylons.config['exec_thread_sleep_time'])) self._updateProgress(maxProgress)
def _runExecThreads(self, execThreads, minProgress, maxProgress): """ run exec threads """ if (len(execThreads) == 0): self._updateProgress(maxProgress) return # start with min progress self._updateProgress(minProgress) # now wait for the threads to complete and update progress firstTime = True while (True): self._checkStop() progress = 0 running = False for (execThread, dummy) in execThreads: #first time, push recovery stack if firstTime: self._pushRecStack(execThread.getCmd()) if dummy: continue self.extendTimeout(execThread.getTimeout()) status = AgentThread._runExeThread(self, (execThread, dummy)) progress += status['progress'] firstTime = False if (not running): break # update the progress percent = progress / len(execThreads) self._updateProgress(calcProgress(minProgress, maxProgress, float(percent) / 100)) time.sleep(float(pylons.config['exec_thread_sleep_time'])) self._updateProgress(maxProgress)
def __init__(self, threadMgr, cat=None): AgentThread.__init__(self, threadMgr, cat)
def __init__(self, threadMgr, uuid): """ Constructor """ AgentThread.__init__(self, threadMgr, cat = [uuid], name = 'cancel_agentthread') self.__uuid = uuid
def stop(self): """ stopping client before calling the super method """ LOG.info('STOP download thread is called stopping') AgentThread.stop(self)
def __startDownload(self): """ actual download logic """ try: LOG.info("Starting package download for package %s" % self.__uriDict['package']) # check to see if there's an in progress file, # since PackageMgr guarantees that duplicate threads will not be spawned # for same pkg, assume an existing thread was killed. # attempt to clean up package n move if (os.path.exists(self.__uriDict['inProgressPackagePath'])): LOG.debug( 'In progress file (%s) already exists. Will validate and reattempt download if necessary' % self.__uriDict['inProgressPackagePath']) if os.path.exists(self.__uriDict['packagePath']): if (os.path.exists(self.__uriDict['propPath']) and PackageUtil.validateProp(self.__uriDict['propPath']) and PackageUtil.validatePackage( self.__uriDict['packagePath'], self.__uriDict['propPath'])): msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict[ 'packagePath'] LOG.info(msg) os.utime(self.__uriDict['packagePath'], None) os.utime(self.__uriDict['propPath'], None) self._updateStatus(progress=100) # NOTE: this is a normal exit not an error! return LOG.warning( 'The package already exists. However package prop (%s) failed validation. Downloading package.' % self.__uriDict['propPath']) # Delete all traces of package before beginning download LOG.info('Cleaning up all packages for %s ' % self.__uriDict['packagePath']) PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath'], self.__uriDict['propPath']) AgentThread._updateProgress(self, 0) if not self.__skipProp: # First, download .prop file LOG.info( 'Starting download of prop file %s - %s' % (self.__uriDict['propUri'], self.__uriDict['propPath'])) self.__download_prop_file() try: self.__prop = loadPropFile(self.__uriDict['propPath']) except FileNotFoundError: raise AgentException( Errors.DC_MISSING_PROP_FILE, 'Prop file (%s) unable to read or did not parse' % (self.__uriDict['propPath'])) AgentThread._updateProgress(self, 2) self.__setProgressTimeouts() if self.__uriDict['scheme'] == 'http': # try download 3 times, with random sleep for _ in range(3): try: sotimeout = float( pylons.config['download_thread_sotimeout']) proxies = json.loads( pylons.config['urlgrabber_proxies']) urlgrabber.urlgrab( self.__uriDict['uri'], self.__uriDict['inProgressPackagePath'], checkfunc=None if self.__skipProp else (PackageUtil.validateDownload, (), {}), progress_obj=DownloadProgress(self), throttle=float(pylons.config['package_throttle']), bandwidth=int(pylons.config['package_bandwidth']), keepalive=0, timeout=sotimeout, proxies=proxies) break except Exception as exc: msg = 'Download error %s - %s' % ( str(exc), traceback.format_exc(3)) LOG.warning(msg) randsleep = randint(30, 60) time.sleep(randsleep) else: # oops! only http and bittorrent supported now raise AgentException( Errors.DC_UNSUPPORTED_PROTOCOL, 'Only http protocols is supported at the moment') self._checkStop() if not self.__skipProp: if (not PackageUtil.validatePackage( self.__uriDict['inProgressPackagePath'], self.__uriDict['propPath'])): raise AgentException( Errors.DC_FAILED_VALIDATE, 'Package ' + self.__uriDict['packagePath'] + ' failed validation') os.utime(self.__uriDict['propPath'], None) utils.rchmod(self.__uriDict['propPath'], "777", 'no') LOG.info( 'Download complete, will now rename and do validation on this file %s' % self.__uriDict['packagePath']) os.rename(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath']) os.utime(self.__uriDict['packagePath'], None) utils.rchmod(self.__uriDict['packagePath'], "777", 'no') LOG.info( "Download complete, Validation completed, updating progress to 100" ) self._updateStatus(progress=100) except AgentException, exc: self._updateStatus(httpStatus=500, progress=0, error=exc.getCode(), errorMsg=exc.getMsg()) msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3)) LOG.error(msg) raise exc
def __init__(self, threadMgr, package): """ Constructor """ AgentThread.__init__(self, threadMgr, name='validate_package') self.__package = package
def __init__(self, threadMgr, cat = None, name = 'agent_thread', parentId = None): """ Constructor """ AgentThread.__init__(self, threadMgr, cat = cat, name = name, parentId = parentId)
def __init__(self, threadMgr, package, parentId = None): """ Constructor """ AgentThread.__init__(self, threadMgr, name = 'validate_package', parentId = parentId) self.__package = package
def __init__(self, threadMgr, module, parentId = None): """ Constructor """ AgentThread.__init__(self, threadMgr, cat = [manifestutil.serviceCat('agent')], name = 'delete_module', parentId = parentId) self.__module = module self.__service = 'agent'
def __init__(self, threadMgr, cat = None): AgentThread.__init__(self, threadMgr, cat)
def __init__(self, threadMgr, progress_queue): AgentThread.__init__(self, threadMgr, cat=['Service/test_status']) self.progress_queue = progress_queue
def __init__(self, threadMgr, service, manifest, parentId = None): """ Constructor """ AgentThread.__init__(self, threadMgr, cat = [manifestutil.serviceCat(service)], name = 'delete_manifest', parentId = parentId) self.service = service self.manifest = manifest
def __init__(self, threadMgr, stopOnly=False): """ Constructor """ AgentThread.__init__(self, threadMgr, cat = [ServicesCleanup.CAT], name = 'cleanup_service') self.__killTimeout = configutil.getConfigAsInt('agent_thread_timeout') #kill -9 should take no time self.__stopOnly = stopOnly
def __init__(self, threadMgr, cat = ['service/test']): AgentThread.__init__(self, threadMgr, cat = cat)
def __init__(self, threadMgr, cat=['service/test']): AgentThread.__init__(self, threadMgr, cat=cat)
def __startDownload(self): """ actual download logic """ try: LOG.info("Starting package download for package %s" % self.__uriDict['package']) # check to see if there's an in progress file, # since PackageMgr guarantees that duplicate threads will not be spawned # for same pkg, assume an existing thread was killed. # attempt to clean up package n move if (os.path.exists(self.__uriDict['inProgressPackagePath'])): LOG.debug('In progress file (%s) already exists. Cleanup and reattempt download' % self.__uriDict['inProgressPackagePath']) if os.path.exists(self.__uriDict['packagePath']): if self.__skipProp or ((os.path.exists(self.__uriDict['propPath']) and PackageUtil.validateProp(self.__uriDict['propPath']) and PackageUtil.validatePackage(self.__uriDict['packagePath'], self.__uriDict['propPath']))): msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict['packagePath'] LOG.info(msg) os.utime(self.__uriDict['packagePath'], None) if os.path.exists(self.__uriDict['propPath']): os.utime(self.__uriDict['propPath'], None) self._updateStatus(progress = 100) # NOTE: this is a normal exit not an error! return else: LOG.warning('The package already exists. However package prop (%s) failed validation.' % self.__uriDict['propPath']) # Delete all traces of package before beginning download LOG.debug('Cleaning up all packages for %s ' % self.__uriDict['packagePath']) PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath'], self.__uriDict['propPath']) AgentThread._updateProgress(self, 0) if self.__skipProp: LOG.info('Skip download of prop file') else: # First, download .prop file LOG.info('Starting download of prop file %s - %s' % (self.__uriDict['propUri'], self.__uriDict['propPath'])) self.__download_prop_file() try: self.__prop = loadPropFile(self.__uriDict['propPath']) except FileNotFoundError: raise AgentException(Errors.DC_MISSING_PROP_FILE, 'Prop file (%s) unable to read or did not parse' % (self.__uriDict['propPath'])) AgentThread._updateProgress(self, 2) self.__setProgressTimeouts() if self.__uriDict['scheme'] == 'http': # try download 3 times, with random sleep attempt = configutil.getConfigAsInt('download_thread_attempt') for _ in range(attempt): try: sotimeout = float(pylons.config['download_thread_sotimeout']) proxies = json.loads(pylons.config['urlgrabber_proxies']) urlgrabber.urlgrab(self.__uriDict['uri'], self.__uriDict['inProgressPackagePath'], # checkfunc = None if self.__skipProp else (PackageUtil.validateDownload, (), {}), progress_obj = DownloadProgress(self), throttle = float(pylons.config['package_throttle']), bandwidth = int(pylons.config['package_bandwidth']), keepalive = 0, timeout = sotimeout, proxies = proxies) break except Exception as exc: msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3)) LOG.warning(msg) if _ == attempt-1: raise exc randsleep = randint(5, 10) time.sleep(randsleep) else: # oops! only http supported now raise AgentException(Errors.DC_UNSUPPORTED_PROTOCOL, 'Only http protocols is supported at the moment') self._checkStop() if self.__skipProp: LOG.info('Skip validating against prop file') else: if (not PackageUtil.validatePackage(self.__uriDict['inProgressPackagePath'], self.__uriDict['propPath'])): raise AgentException(Errors.DC_FAILED_VALIDATE, 'Package ' + self.__uriDict['packagePath'] + ' failed validation') os.utime(self.__uriDict['propPath'], None) utils.rchmod(self.__uriDict['propPath'], "777", 'no') LOG.info('Download complete, rename this file %s' % self.__uriDict['packagePath']) os.rename(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath']) os.utime(self.__uriDict['packagePath'], None) utils.rchmod(self.__uriDict['packagePath'], "777", 'no') LOG.info("Download complete, updating progress to 100") self._updateStatus(progress = 100) except AgentException, exc: self._updateStatus(httpStatus = 500, progress = 0, error = exc.getCode(), errorMsg = exc.getMsg()) msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3)) LOG.error(msg) raise exc
def __init__(self, threadMgr, cat = None, name = 'agent_thread'): """ Constructor """ AgentThread.__init__(self, threadMgr, cat = cat, name = name)
def __init__(self, threadMgr): """ Constructor """ AgentThread.__init__(self, threadMgr, cat = [SafeShutdown.CAT], name = SafeShutdown.THREAD_NAME)
def __init__(self, threadMgr): """ Constructor """ AgentThread.__init__(self, threadMgr, cat=[SafeShutdown.CAT], name=SafeShutdown.THREAD_NAME)
def __init__(self, threadMgr, service, path): """ Constructor """ AgentThread.__init__(self, threadMgr, cat = [manifestutil.serviceCat(service)], name = 'delete_service') self.__path = path self.service = service