def _untarPackages(self, packages, service, untarRootPath, nicelevel=None, pgksNeedSuffix=None, pathSuffix=''): """ untar all the packages @params packages - list of dictionary of parsed packages uri @throws AgentException """ self._updateProgress(80) count = 0 for pkgDict in packages.itervalues(): count += 1 # check if package path exists already # if the package is already untarred, move on # else create the package path pkgName = pkgDict['packageName'] untarPath = os.path.join(untarRootPath, pkgName, '%s%s' % (pkgDict['packageVersion'], pathSuffix if ((pgksNeedSuffix is not None) and pkgName in pgksNeedSuffix) else '')) if os.path.exists(untarPath): # perhaps another thread has finished extraction, continue with next package continue os.makedirs(untarPath) try: self.untar(pkgDict['packagePath'], untarPath, nicelevel) except AgentException: rmrf(untarPath) # Note: i. atleast self should have 'rx' so that we can proceed setting 'rx' for group and others # ii. since cronus/cronusapp belong to diff group, lets give access to self, group and others. # if both belong to same group in future, then just self, group should be enough if (not os.name == 'nt'): # ensure all parent dir of scripts dir have 'rx' so that we can really navigate to scripts dir and execute uname = getuserofpath(untarPath) chmod(untarPath, '+rx', sudoUser = uname) cronusPath = os.path.join(untarPath, 'cronus') if os.path.exists(cronusPath): uname = getuserofpath(cronusPath) chmod(cronusPath, '+rx', sudoUser = uname) # now give all scripts 'rx' permission scriptsPath = os.path.join(cronusPath, 'scripts') if os.path.exists(scriptsPath): uname = getuserofpath(scriptsPath) rchmod(scriptsPath, '+rx', sudoUser = uname) #Running as cronus user when higher privilege service (i.e. not chown all the package into the application user) if (not isHigherPrivilegeService(service)): import pwd uname = pylons.config['app_user_account'] uid = pwd.getpwnam(uname).pw_uid gid = pwd.getpwnam(uname).pw_gid rchown(untarPath, uid, gid) self._updateProgress(calcProgress(80, 99, float(count) / len(packages)))
def post(self, service): """ Create a new service object """ try: LOG.info('Got a post request for service ' + service) path = ServiceController.servicePath(service) # skip if the path already exists if os.path.exists(path) and os.path.isdir(path): return doneResult(request, response, httpStatus=201, result=service, controller=self) os.makedirs(path) os.makedirs(os.path.join(path, 'manifests')) os.makedirs(os.path.join(path, 'installed-packages')) os.makedirs(os.path.join(path, 'modules')) os.makedirs(os.path.join(path, 'downloaded-packages')) os.makedirs(os.path.join(path, '.appdata')) os.makedirs(os.path.join(path, '.data')) utils.rchmod(os.path.join(path, '.appdata'), '+rwx') # verify that the path exists if (not os.path.isdir(path)): return errorResult(request, response, Errors.UNKNOWN_ERROR, "Service(%s) was not created" % service, controller=self) return doneResult(request, response, result=service, controller=self) except OSError: return errorResult(request, response, Errors.SERVICE_EXISTS, "Service(%s) already exists(or %s)" % (service, traceback.format_exc(2)), controller=self) except Exception as excep: return errorResult( request, response, error=Errors.UNKNOWN_ERROR, errorMsg='Unknown error when posting services %s - %s' % (str(excep), traceback.format_exc(2)), controller=self)
def post(self, service): """ Create a new service object """ try: LOG.info("Got a post request for service " + service) path = ServiceController.servicePath(service) # skip if the path already exists if os.path.exists(path) and os.path.isdir(path): return doneResult(request, response, httpStatus=201, result=service, controller=self) os.makedirs(path) os.makedirs(os.path.join(path, "manifests")) os.makedirs(os.path.join(path, "installed-packages")) os.makedirs(os.path.join(path, "modules")) os.makedirs(os.path.join(path, "downloaded-packages")) os.makedirs(os.path.join(path, ".appdata")) os.makedirs(os.path.join(path, ".data")) utils.rchmod(os.path.join(path, ".appdata"), "+rwx") # verify that the path exists if not os.path.isdir(path): return errorResult( request, response, Errors.UNKNOWN_ERROR, "Service(%s) was not created" % service, controller=self ) return doneResult(request, response, result=service, controller=self) except OSError: return errorResult( request, response, Errors.SERVICE_EXISTS, "Service(%s) already exists(or %s)" % (service, traceback.format_exc(2)), controller=self, ) except Exception as excep: return errorResult( request, response, error=Errors.UNKNOWN_ERROR, errorMsg="Unknown error when posting services %s - %s" % (str(excep), traceback.format_exc(2)), controller=self, )
def __startDownload(self): """ actual download logic """ try: LOG.info("Starting package download for package %s" % self.__uriDict['package']) # check to see if there's an in progress file, # since PackageMgr guarantees that duplicate threads will not be spawned # for same pkg, assume an existing thread was killed. # attempt to clean up package n move if (os.path.exists(self.__uriDict['inProgressPackagePath'])): LOG.debug( 'In progress file (%s) already exists. Will validate and reattempt download if necessary' % self.__uriDict['inProgressPackagePath']) if os.path.exists(self.__uriDict['packagePath']): if (os.path.exists(self.__uriDict['propPath']) and PackageUtil.validateProp(self.__uriDict['propPath']) and PackageUtil.validatePackage( self.__uriDict['packagePath'], self.__uriDict['propPath'])): msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict[ 'packagePath'] LOG.info(msg) os.utime(self.__uriDict['packagePath'], None) os.utime(self.__uriDict['propPath'], None) self._updateStatus(progress=100) # NOTE: this is a normal exit not an error! return LOG.warning( 'The package already exists. However package prop (%s) failed validation. Downloading package.' % self.__uriDict['propPath']) # Delete all traces of package before beginning download LOG.info('Cleaning up all packages for %s ' % self.__uriDict['packagePath']) PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath'], self.__uriDict['propPath']) AgentThread._updateProgress(self, 0) if not self.__skipProp: # First, download .prop file LOG.info( 'Starting download of prop file %s - %s' % (self.__uriDict['propUri'], self.__uriDict['propPath'])) self.__download_prop_file() try: self.__prop = loadPropFile(self.__uriDict['propPath']) except FileNotFoundError: raise AgentException( Errors.DC_MISSING_PROP_FILE, 'Prop file (%s) unable to read or did not parse' % (self.__uriDict['propPath'])) AgentThread._updateProgress(self, 2) self.__setProgressTimeouts() if self.__uriDict['scheme'] == 'http': # try download 3 times, with random sleep for _ in range(3): try: sotimeout = float( pylons.config['download_thread_sotimeout']) proxies = json.loads( pylons.config['urlgrabber_proxies']) urlgrabber.urlgrab( self.__uriDict['uri'], self.__uriDict['inProgressPackagePath'], checkfunc=None if self.__skipProp else (PackageUtil.validateDownload, (), {}), progress_obj=DownloadProgress(self), throttle=float(pylons.config['package_throttle']), bandwidth=int(pylons.config['package_bandwidth']), keepalive=0, timeout=sotimeout, proxies=proxies) break except Exception as exc: msg = 'Download error %s - %s' % ( str(exc), traceback.format_exc(3)) LOG.warning(msg) randsleep = randint(30, 60) time.sleep(randsleep) else: # oops! only http and bittorrent supported now raise AgentException( Errors.DC_UNSUPPORTED_PROTOCOL, 'Only http protocols is supported at the moment') self._checkStop() if not self.__skipProp: if (not PackageUtil.validatePackage( self.__uriDict['inProgressPackagePath'], self.__uriDict['propPath'])): raise AgentException( Errors.DC_FAILED_VALIDATE, 'Package ' + self.__uriDict['packagePath'] + ' failed validation') os.utime(self.__uriDict['propPath'], None) utils.rchmod(self.__uriDict['propPath'], "777", 'no') LOG.info( 'Download complete, will now rename and do validation on this file %s' % self.__uriDict['packagePath']) os.rename(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath']) os.utime(self.__uriDict['packagePath'], None) utils.rchmod(self.__uriDict['packagePath'], "777", 'no') LOG.info( "Download complete, Validation completed, updating progress to 100" ) self._updateStatus(progress=100) except AgentException, exc: self._updateStatus(httpStatus=500, progress=0, error=exc.getCode(), errorMsg=exc.getMsg()) msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3)) LOG.error(msg) raise exc
def __startDownload(self): """ actual download logic """ try: LOG.info("Starting package download for package %s" % self.__uriDict['package']) # check to see if there's an in progress file, # since PackageMgr guarantees that duplicate threads will not be spawned # for same pkg, assume an existing thread was killed. # attempt to clean up package n move if (os.path.exists(self.__uriDict['inProgressPackagePath'])): LOG.debug('In progress file (%s) already exists. Cleanup and reattempt download' % self.__uriDict['inProgressPackagePath']) if os.path.exists(self.__uriDict['packagePath']): if self.__skipProp or ((os.path.exists(self.__uriDict['propPath']) and PackageUtil.validateProp(self.__uriDict['propPath']) and PackageUtil.validatePackage(self.__uriDict['packagePath'], self.__uriDict['propPath']))): msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict['packagePath'] LOG.info(msg) os.utime(self.__uriDict['packagePath'], None) if os.path.exists(self.__uriDict['propPath']): os.utime(self.__uriDict['propPath'], None) self._updateStatus(progress = 100) # NOTE: this is a normal exit not an error! return else: LOG.warning('The package already exists. However package prop (%s) failed validation.' % self.__uriDict['propPath']) # Delete all traces of package before beginning download LOG.debug('Cleaning up all packages for %s ' % self.__uriDict['packagePath']) PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath'], self.__uriDict['propPath']) AgentThread._updateProgress(self, 0) if self.__skipProp: LOG.info('Skip download of prop file') else: # First, download .prop file LOG.info('Starting download of prop file %s - %s' % (self.__uriDict['propUri'], self.__uriDict['propPath'])) self.__download_prop_file() try: self.__prop = loadPropFile(self.__uriDict['propPath']) except FileNotFoundError: raise AgentException(Errors.DC_MISSING_PROP_FILE, 'Prop file (%s) unable to read or did not parse' % (self.__uriDict['propPath'])) AgentThread._updateProgress(self, 2) self.__setProgressTimeouts() if self.__uriDict['scheme'] == 'http': # try download 3 times, with random sleep attempt = configutil.getConfigAsInt('download_thread_attempt') for _ in range(attempt): try: sotimeout = float(pylons.config['download_thread_sotimeout']) proxies = json.loads(pylons.config['urlgrabber_proxies']) urlgrabber.urlgrab(self.__uriDict['uri'], self.__uriDict['inProgressPackagePath'], # checkfunc = None if self.__skipProp else (PackageUtil.validateDownload, (), {}), progress_obj = DownloadProgress(self), throttle = float(pylons.config['package_throttle']), bandwidth = int(pylons.config['package_bandwidth']), keepalive = 0, timeout = sotimeout, proxies = proxies) break except Exception as exc: msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3)) LOG.warning(msg) if _ == attempt-1: raise exc randsleep = randint(5, 10) time.sleep(randsleep) else: # oops! only http supported now raise AgentException(Errors.DC_UNSUPPORTED_PROTOCOL, 'Only http protocols is supported at the moment') self._checkStop() if self.__skipProp: LOG.info('Skip validating against prop file') else: if (not PackageUtil.validatePackage(self.__uriDict['inProgressPackagePath'], self.__uriDict['propPath'])): raise AgentException(Errors.DC_FAILED_VALIDATE, 'Package ' + self.__uriDict['packagePath'] + ' failed validation') os.utime(self.__uriDict['propPath'], None) utils.rchmod(self.__uriDict['propPath'], "777", 'no') LOG.info('Download complete, rename this file %s' % self.__uriDict['packagePath']) os.rename(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath']) os.utime(self.__uriDict['packagePath'], None) utils.rchmod(self.__uriDict['packagePath'], "777", 'no') LOG.info("Download complete, updating progress to 100") self._updateStatus(progress = 100) except AgentException, exc: self._updateStatus(httpStatus = 500, progress = 0, error = exc.getCode(), errorMsg = exc.getMsg()) msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3)) LOG.error(msg) raise exc
def executeScript(self): """ execute a script from remote location""" scriptpath = None try: # parse the body if (not request.body or request.body == ""): LOG.error('invalid body found in post command') return errorResult(request, response, 10001, 'No body found in post command', controller = self) body = json.loads(request.body.encode('ascii', 'ignore')) scriptloc = body['scriptLocation'] if 'scriptLocation' in body else None scriptname = body['scriptName'] if 'scriptName' in body else None hasSudo = ('sudoUser' in body and body['sudoUser']) sudoUser = body['sudoUser'] if ('sudoUser' in body and body['sudoUser'] != 'root') else None paramobj = body['params'] if 'params' in body else [] params = paramobj if type(paramobj) == list else paramobj.split() LOG.info('%s %s %s %s %s' % (scriptloc, scriptname, hasSudo, sudoUser, params)) if not scriptloc: return errorResult(request, response, Errors.INVALID_REQUEST, 'Script location not found', controller = self) if not scriptname: return errorResult(request, response, Errors.INVALID_REQUEST, 'Script name not found', controller = self) scriptpath = os.path.join(self.dataPath(), scriptname) LOG.info('scriptpath = %s' % scriptpath) utils.runsyscmd('wget %s -O %s' % (scriptloc, scriptpath)) if not os.path.exists(scriptpath): return errorResult(request, response, Errors.FILE_NOT_FOUND_ERROR, 'Failed to get script %s' % scriptpath, controller = self) utils.rchmod(scriptpath, '+rx') cmd = [scriptpath] if hasSudo: cmd.insert(0, 'sudo') if sudoUser: cmd.insert(1, sudoUser) cmd.insert(1, '-u') for param in params: cmd.append(param) LOG.info('cmd = %s' % cmd) appGlobal = config['pylons.app_globals'] execThread = ExecThread(appGlobal.threadMgr, cmd) execThread.setLogLevel('info') contextutils.copyJobContexts(self, execThread) execThread.start() execThread.threadMgrEvent.wait() return statusResult(request, response, execThread, controller = self) except Exception as excp: return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg = 'Unknown error when executing cmd %s, %s - %s' % (scriptpath, str(excp), traceback.format_exc(2)), controller = self)
def _untarPackages(self, packages, service, untarRootPath, nicelevel=None, pgksNeedSuffix=None, pathSuffix=''): """ untar all the packages @params packages - list of dictionary of parsed packages uri @throws AgentException """ self._updateProgress(80) count = 0 for pkgDict in packages.itervalues(): count += 1 # check if package path exists already # if the package is already untarred, move on # else create the package path pkgName = pkgDict['packageName'] untarPath = os.path.join(untarRootPath, pkgName, '%s%s' % (pkgDict['packageVersion'], pathSuffix if ((pgksNeedSuffix is not None) and pkgName in pgksNeedSuffix) else '')) if os.path.exists(untarPath): # perhaps another thread has finished extraction, continue with next package continue os.makedirs(untarPath) try: self.untar(pkgDict['packagePath'], untarPath, nicelevel) except AgentException: rmrf(untarPath) # Note: i. atleast self should have 'rx' so that we can proceed setting 'rx' for group and others # if both belong to same group in future, then just self, group should be enough # ensure all parent dir of scripts dir have 'rx' so that we can really navigate to scripts dir and execute uname = getuserofpath(untarPath) chmod(untarPath, '+rx', sudoUser = uname) cronusPath = os.path.join(untarPath, 'cronus') if os.path.exists(cronusPath): uname = getuserofpath(cronusPath) chmod(cronusPath, '+rx', sudoUser = uname) # now give all scripts 'rx' permission scriptsPath = os.path.join(cronusPath, 'scripts') if os.path.exists(scriptsPath): uname = getuserofpath(scriptsPath) rchmod(scriptsPath, '+rx', sudoUser = uname) # issue #16, now add symlink to .appdata for easy access appdata_path = manifestutil.appDataPath(service) link_path = os.path.join(untarPath, '.appdata') LOG.info('Create .appdata symlink from %s to %s' % (link_path, appdata_path)) utils.symlink(appdata_path, link_path) #Running as cronus user when higher privilege service (i.e. not chown all the package into the application user) if (not isHigherPrivilegeService(service)): uname = configutil.getAppUser() uid, gid = utils.getUidGid(uname) rchown(untarPath, uid, gid) self._updateProgress(calcProgress(80, 99, float(count) / len(packages)))
def executeScript(self): """ execute a script from remote location""" scriptpath = None try: # parse the body if (not request.body or request.body == ""): LOG.error('invalid body found in post command') return errorResult(request, response, 10001, 'No body found in post command', controller=self) body = json.loads(request.body) scriptloc = body['script-location'].encode( 'ascii', 'ignore') if 'script-location' in body else None scriptname = body['script-name'].encode( 'ascii', 'ignore') if 'script-name' in body else None needsudo = asbool( body['need-sudo']) if 'need-sudo' in body else False sudotgt = body['sudo-target'].encode( 'ascii', 'ignore') if 'sudo-target' in body else None paramobj = body['params'] if 'params' in body else [] params = paramobj if type(paramobj) == list else paramobj.split() LOG.info('%s %s %s %s %s' % (scriptloc, scriptname, needsudo, sudotgt, params)) if scriptloc is None or scriptloc == '': return errorResult(request, response, 10003, 'Script location not found', controller=self) if scriptname is None or scriptname == '': return errorResult(request, response, 10003, 'Script name not found', controller=self) scriptpath = os.path.join(self.dataPath(), scriptname) LOG.info('scriptpath = %s' % scriptpath) os.system('wget %s -O %s' % (scriptloc, scriptpath)) if scriptpath is None or not os.path.exists(scriptpath): return errorResult(request, response, 10003, 'Failed to get script %s' % scriptpath, controller=self) rchmod(scriptpath, '+rx') cmd = [scriptpath] if needsudo: cmd.insert(0, 'sudo') if sudotgt is not None: cmd.insert(1, sudotgt) cmd.insert(1, '-u') for param in params: param = param.encode('ascii', 'ignore') cmd.append(param) LOG.info('cmd = %s' % cmd) appGlobal = config['pylons.app_globals'] execThread = ExecThread(appGlobal.threadMgr, cmd) execThread.setLogLevel('info') execThread.start() execThread.threadMgrEvent.wait() return statusResult(request, response, execThread, controller=self) except Exception as excp: return errorResult( request, response, error=Errors.UNKNOWN_ERROR, errorMsg='Unknown error when executing cmd %s, %s - %s' % (scriptpath, str(excp), traceback.format_exc(2)), controller=self)