def testServiceFromPath(self): servicepath = manifestutil.servicePath('foo') print servicepath, manifestutil.serviceFromPath(servicepath) assert 'foo' == manifestutil.serviceFromPath(servicepath) servicepath = manifestutil.servicePath('.envfoo.poolbar.machinebaz') assert '.envfoo.poolbar.machinebaz' == manifestutil.serviceFromPath(servicepath) manifestpath = manifestutil.manifestPath('foo', 'bar') assert 'foo' == manifestutil.serviceFromPath(manifestpath) manifestpaths = ['somedummypath', manifestpath] assert 'foo' == manifestutil.serviceFromPath(manifestpaths)
def testServiceFromPath(self): servicepath = manifestutil.servicePath('foo') print servicepath, manifestutil.serviceFromPath(servicepath) assert 'foo' == manifestutil.serviceFromPath(servicepath) servicepath = manifestutil.servicePath('.envfoo.poolbar.machinebaz') assert '.envfoo.poolbar.machinebaz' == manifestutil.serviceFromPath( servicepath) manifestpath = manifestutil.manifestPath('foo', 'bar') assert 'foo' == manifestutil.serviceFromPath(manifestpath) manifestpaths = ['somedummypath', manifestpath] assert 'foo' == manifestutil.serviceFromPath(manifestpaths)
def dataPath(self): """ compute the path to the packages """ datapath = os.path.realpath( os.path.join(manifestutil.servicePath('agent'), '.data')) if not os.path.exists(datapath): os.system('mkdir %s' % datapath) return datapath
def test_getAllInstalledPackages(self): from agent.tests.unit.test_util import createManifest serviceName = 'service1' manifestName = 'manifestA' try: createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"], service = serviceName, manifest = manifestName) except Exception as ex: print 'exception thrown during mf %s' % str(ex) time.sleep(2) installedPkgPath = os.path.join(manifestutil.servicePath(serviceName), 'installed-packages') pkgs = PackageUtil.getAllInstalledPackages(installedPkgPath) self.assertEquals(pkgs.__len__(), 2)
def cleanRogueService(): """ delete rogue services """ try: services = manifestutil.getServices() for service in services: path = manifestutil.servicePath(service) for idx in range(3): if os.path.exists(os.path.join(path, 'manifests')): break time.sleep(2) if idx == 2: appGlobal = pylons.config['pylons.app_globals'] LOG.info('service %s does not have manifests folder, cleanup the rogue service' % service) deleteThread = ServiceDelete(appGlobal.threadMgr, service, path) deleteThread.run() LOG.info('service %s cleaned up' % service) break except Exception: LOG.error('failed to check and cleanup rogue service' + traceback.format_exc(5))
def __deleteService(self, service): """ delete all services except agent itself; clear all manifests in 'agent' itself except current active""" self._updateStatus(progress = 60) #remove folder path = manifestutil.servicePath(service) # retry service cleanup for _ in range(3): if not os.path.exists(path): break ServiceDelete.deleteFolderContents(path) # sleep here a bit to ensure delete is complete time.sleep(1) if os.path.exists(path): msg = 'Could not delete service %s completely even after 3 retries.' % service LOG.error(msg) raise Exception(msg) self._updateStatus(progress = 90)
def cleanRogueService(): """ delete rogue services """ try: services = manifestutil.getServices() for service in services: path = manifestutil.servicePath(service) for idx in range(3): if os.path.exists(os.path.join(path, 'manifests')): break time.sleep(2) if idx == 2: appGlobal = pylons.config['pylons.app_globals'] LOG.info( 'service %s does not have manifests folder, cleanup the rogue service' % service) deleteThread = ServiceDelete(appGlobal.threadMgr, service, path) deleteThread.run() LOG.info('service %s cleaned up' % service) break except Exception: LOG.error('failed to check and cleanup rogue service' + traceback.format_exc(5))
def doRun(self): """ run - exec the thread and parse the results """ display_cmd = '' try: self.__LOG.debug("Exec Thread running %s" % self.__cmd) closeFds = readall = True service = manifestutil.serviceFromPath(self.__cmd) cronusapp_home = '' if (service != None and len(service) > 0): cronusapp_home = str(manifestutil.servicePath(service)) elif (contextutils.existcontext(self, 'service')): service = str(contextutils.getcontext(self, 'service')) cronusapp_home = str(manifestutil.servicePath(service)) correlation_id = str(contextutils.getcontext(self, 'guid', '')) env_variables = 'CRONUSAPP_HOME=%s LCM_CORRELATIONID=%s' % (cronusapp_home, correlation_id) if isinstance(self.__cmd, basestring): cmd_0 = str(self.__cmd) if (cmd_0.find('sudo -u') >= 0): # as we have problem setting environment variables for sudo as root cmd_0 = cmd_0.replace("sudo", ("sudo %s" % env_variables), 1) self.__cmd = cmd_0 display_cmd = cmd_0.split(os.path.sep)[-1] else: cmd_0 = self.__cmd[0] if (cmd_0 == 'sudo' and '-u' in self.__cmd): # as we have problem setting environment variables for sudo as root for env_variable in env_variables.split(' ')[::-1]: self.__cmd.insert(1, env_variable) display_cmd = self.__cmd[-1].split(os.path.sep)[-1] threadName = 'exec_thread(%s)' % display_cmd self.__cmdProcess = subprocess.Popen(self.__cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, close_fds = closeFds) # in case we don't get any results self.__response = {'progress': 100, 'result': None} if 'info' == self.__logLevel: self.__LOG.info("started Exec Thread..., cmd=%s" % self.__cmd) outputlines = [] while True: self._checkStop(threadName = threadName) stopped = (self.__cmdProcess.poll() != None) # read from stdout, if process completes, read all from stdout lines = self.readStream(self.__cmdProcess.stdout, readall = readall) if (not lines and stopped): break outputlines.extend(lines) if 'info' == self.__logLevel and lines: self.__LOG.info('cmd(%s) output:' % display_cmd) for line in lines: self.__LOG.info(line) else: self.__LOG.debug('cmd(%s) output:' % display_cmd) for line in lines: self.__LOG.debug(line) for line in lines: self.processExecResponse(line) if (not lines): time.sleep(float(config['exec_thread_sleep_time'])) if 'info' == self.__logLevel and outputlines: OUTPUTLOG.info('%s output start %s %s output end' % (self.getUuid(), '\\n'.join(outputlines), self.getUuid())) # maybe the script just closed stdout # wait until the script finishes while (self.__cmdProcess.poll() == None): self._checkStop(threadName = threadName) time.sleep(float(config['exec_thread_sleep_time'])) returnCode = self.__cmdProcess.poll() # the error condition if (returnCode != 0): self._updateStatus(httpStatus = 500) # read from stderr and log if self.__cmdProcess.stderr is not None: lines = self.readStream(self.__cmdProcess.stderr) for line in lines: self.processExecResponse(line) self.__LOG.info('cmd(%s) stderr: %s' % (display_cmd, line)) self.__LOG.warning(self.__response) errorCode = int(self.__response['error']) if 'error' in self.__response else returnCode msg = 'Application script error (%s) error code (%d) error msg (%s)' % (self.__cmd, errorCode, (self.__response['errorMsg'] if 'errorMsg' in self.__response else '')) self.__LOG.warning(msg) # now add 16000 to the error code to indicate this is a client error clientErrorCode = Errors.CLIENT_SCRIPT_ERROR + abs(errorCode) self._updateStatus(error = clientErrorCode, errorMsg = msg) else: self._updateStatus(httpStatus = 200) if 'result' in self.__response: self._updateStatus(result = self.__response['result']) self._updateStatus(progress = 100) except SystemExit as excep: # status already set in agent thread msg = "System Exception for cmd %s - %s" % (self.__cmd, self.getStatus()['errorMsg']) self.__LOG.warning("%s - %s" % (msg, str(excep))) raise excep except OSError as excep: if self.__cmdProcess is None: msg = 'Cannot create subprocess cmd(%s) %s - %s' % (self.__cmd, str(excep), traceback.format_exc(2)) else: msg = 'Unknown OSError cmd(%s) %s - %s' % (self.__cmd, str(excep), traceback.format_exc(2)) self.__LOG.warning(msg) self._updateStatus(httpStatus = 500, error = Errors.UNKNOWN_ERROR, errorMsg = msg) except Exception as excep: msg = 'Unknown error cmd(%s) %s - %s' % (self.__cmd, str(excep), traceback.format_exc(2)) self.__LOG.warning(msg) self._updateStatus(httpStatus = 500, error = Errors.UNKNOWN_ERROR, errorMsg = msg) finally: if self.__cmdProcess: if (self.__cmdProcess.stdout != None): self.__cmdProcess.stdout.close() if (self.__cmdProcess.stdin != None): self.__cmdProcess.stdin.close() if (self.__cmdProcess.stderr != None): self.__cmdProcess.stderr.close() self.kill(self.__cmdProcess, self.__cmd) self.__LOG.debug("Exec Thread %s done" % self.__cmd)
def testGetServices(self): createManifest(self, manifest = 'bar', service = 'foo') assert 'foo' in manifestutil.getServices() assert manifestutil.servicePath('foo').endswith('foo')
def testServiceMetadataPath(self): service = '.env.service.instance' servicepath = manifestutil.servicePath(service) metadatapath = manifestutil.serviceMetadataPath(service) assert metadatapath == (servicepath + os.path.sep + '.metadata.json')
def startServicesOnAgentStartup(): """ when agent is restarted, 0. check for agent selfupdate 1. start all service with active manifest, this requires service startup script be idempotent 2. load dynamic controllers and routes if any """ # check for agent update from agent.lib.agenthealth import checkAgentVersion checkAgentVersion(True) # startup services from agent.lib.agent_thread.startstop_service import StartStopService appGlobal = config['pylons.app_globals'] appdelay = int( config['appinitdelay']) if 'appinitdelay' in config else 0 if appdelay > 0: time.sleep(appdelay) # check if this is agent restart or system restart if os.name != 'nt' and os.path.exists("/proc/uptime"): uptime, _ = [float(f) for f in open("/proc/uptime").read().split()] else: uptime = 500 systemRestartTimeThreshold = pylons.config[ 'system_restart_time_threshold'] actionType = StartStopService.ACTION_RESTART if (int(systemRestartTimeThreshold) > uptime): actionType = StartStopService.ACTION_REBOOT for service in manifestutil.getServices(): appDataDir = manifestutil.appDataPath(service) if not os.path.exists(appDataDir): os.makedirs(appDataDir) import pwd uname = pylons.config['app_user_account'] uid = pwd.getpwnam(uname).pw_uid gid = pwd.getpwnam(uname).pw_gid utils.rchown(appDataDir, uid, gid) dataDir = manifestutil.dataPath(service) if not os.path.exists(dataDir): os.makedirs(dataDir) if service != 'agent': try: manifestutil.updateServiceMetaFile( service, { 'servicePath': manifestutil.servicePath(service), 'serviceName': service }) except Exception as excep: LOG.error( 'Unknown error updating local metadata service(%s) - %s - %s' % (service, str(excep), traceback.format_exc(2))) if manifestutil.hasActiveManifest(service): try: LOG.info('startup for service(%s)', service) startupThread = StartStopService( appGlobal.threadMgr, service, actionType) startupThread.start() startupThread.threadMgrEvent.wait() except Exception as excep: LOG.error( 'Unknown error starting service(%s) - %s - %s' % (service, str(excep), traceback.format_exc(2))) try: LOG.info('load controllers and routes for service(%s)', service) manifestutil.processControllerInPackage(service) except Exception as excep: LOG.error( 'Unknown error loading controllers for service(%s) - %s - %s' % (service, str(excep), traceback.format_exc(2)))
def testGetServices(self): createManifest(self, manifest='bar', service='foo') assert 'foo' in manifestutil.getServices() assert manifestutil.servicePath('foo').endswith('foo')
def doRun(self): """ Main body of the thread """ spath = manifestutil.servicePath(self._service) self._updateProgress(1) errorMsg = "" errorCode = None failed = False ctxNames = ['guid', 'service'] try: # create service if not already exist if not os.path.exists(spath): os.makedirs(spath) os.makedirs(os.path.join(spath, 'manifests')) os.makedirs(os.path.join(spath, 'installed-packages')) os.makedirs(os.path.join(spath, 'modules')) os.makedirs(os.path.join(spath, 'downloaded-packages')) os.makedirs(os.path.join(spath, '.appdata')) os.makedirs(os.path.join(spath, '.data')) import pwd uname = pylons.config['app_user_account'] uid = pwd.getpwnam(uname).pw_uid gid = pwd.getpwnam(uname).pw_gid utils.rchown(os.path.join(spath, '.appdata'), uid, gid) # verify that the path exists if (not os.path.isdir(spath)): raise AgentException(Errors.UNKNOWN_ERROR, "Service(%s) was not created" % self._service) self._updateProgress(20) # create manifest if not already exist mpath = manifestutil.manifestPath(self._service, self._manifest) if (not os.path.exists(mpath) or not os.path.isdir(mpath)): self.__LOG.debug('pkgs = %s', self.__packages) # parse the package list for idx, package in enumerate(self.__packages): if package.startswith('/'): packageRef = package tokens = package.split('/') pkgnamePrefix = tokens[-1].rstrip() fullPkgLoc = manifestutil.getPackageByName(self._service, manifest = None, pkgnamePrefix = pkgnamePrefix) if fullPkgLoc is None: raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST, 'manifest (%s/%s) package (%s) does not exist' % (self._service, self._manifest, self.__packages)) else: self.__LOG.info('expanding package reuse ref %s with full package location %s' % (packageRef, fullPkgLoc)) self.__packages[idx] = fullPkgLoc # start a thread to create the package manThread = ManifestCreate(threadmgr.NULL_THREADMGR, self._service, self._manifest, self.__packages, skipProp = self.__skipProp) contextutils.copycontexts(self, manThread, ctxNames) manThread.run() status = manThread.getStatus() if (status['error'] != None): raise AgentException(status['error'], status['errorMsg']) self._updateProgress(60) if (not os.path.exists(mpath)): raise AgentException(Errors.ACTIVEMANIFEST_MANIFEST_MISSING, 'Manifest(%s, %s) path missing' % (self._service, self._manifest)) if not self.__skipActivation: activateThread = ActivateManifest(threadmgr.NULL_THREADMGR, self._service, self._manifest) contextutils.copycontexts(self, activateThread, ctxNames) activateThread.run() status = activateThread.getStatus() if (status['error'] != None): raise AgentException(status['error'], status['errorMsg']) # activte manifest if not already activated if manifestutil.getActiveManifest(self._service) != self._manifest: raise AgentException(Errors.ACTIVEMANIFEST_MANIFEST_MISSING, 'Manifest(%s, %s) path missing' % (self._service, self._manifest)) except SystemExit as exc: failed = True if (len(exc.args) == 2): # ok we got {err code, err msg} errorCode = exc.args[0] errorMsg = exc.args[1] except AgentException as exc: failed = True errorMsg = 'Deploy Service - Agent Exception - %s' % exc.getMsg() errorCode = exc.getCode() except Exception as exc: failed = True errorMsg = 'Deploy Service - Unknown error - (%s/%s) - %s - %s' \ % (self._service, self._manifest, str(exc), traceback.format_exc(5)) errorCode = Errors.UNKNOWN_ERROR finally: if failed: self.__LOG.warning(errorMsg) self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg) else: self._updateProgress(100)
def doRun(self): """ run - exec the thread and parse the results """ display_cmd = '' outputLogged = False try: serviceFromPath = manifestutil.serviceFromPath(self.__cmd, 'agent') self.__service = contextutils.getcontext(self, 'service', serviceFromPath) self.__LOG = manifestutil.getServiceLogger(self, logging.getLogger(__name__), self.__service) self.log("Exec Thread running, cmd=%s" % self.__cmd) closeFds = readall = True # context injection via environment variables if self.__injectctx: cronusapp_home = str(manifestutil.servicePath(self.__service)) correlation_id = str(contextutils.getcontext(self, 'guid', '')) authz_token = str(contextutils.getcontext(self, 'authztoken', '')) env_variables = 'SERVICE_NAME=%s CRONUSAPP_HOME=%s CORRELATIONID=%s AUTHZTOKEN=%s' % ( self.__service, cronusapp_home, correlation_id, authz_token) if self.__service: lcmMetas = serviceutil.getLcmMetas(self.__service) if lcmMetas: for key, value in lcmMetas.items(): env_variables += (' %s=%s' % (key.upper(), value)) if isinstance(self.__cmd, basestring): cmd_0 = str(self.__cmd) if (cmd_0.find('sudo -u') >= 0): # as we have problem setting environment variables for sudo as root cmd_0 = cmd_0.replace("sudo", ("sudo %s" % env_variables), 1) self.__cmd = cmd_0 display_cmd = cmd_0.split(os.path.sep)[-1] else: cmd_0 = self.__cmd[0] if (cmd_0 == 'sudo' and '-u' in self.__cmd): # as we have problem setting environment variables for sudo as root for env_variable in env_variables.split(' ')[::-1]: self.__cmd.insert(1, env_variable) display_cmd = self.__cmd[-1].split(os.path.sep)[-1] threadName = 'exec_thread(%s)' % display_cmd # execute self.__cmdProcess = subprocess.Popen(self.__cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, close_fds = closeFds) self.logExecOutput('%s uuid start ' % (self.getUuid())) outputLogged = True self.logExecOutput('cmd: %s' % (self.__cmd if isinstance(self.__cmd, basestring) else (' '.join(self.__cmd)))) # in case we don't get any results self.__response = {'progress': 0.0, 'result': None} self.log('cmd(%s) output:' % display_cmd) while True: self._checkStop(threadName = threadName) stopped = (self.__cmdProcess.poll() != None) # read from stdout, if process completes, read all from stdout lines = self.readStream(self.__cmdProcess.stdout, readall = readall) if (not lines and stopped): break self.__outputlines.extend(lines) for line in lines: self.processExecResponse(line) self.log(line) if (not lines): time.sleep(float(config['exec_thread_sleep_time'])) if self.__outputlines: self.logExecOutput('\n'.join(self.__outputlines)) # maybe the script just closed stdout # wait until the script finishes while (self.__cmdProcess.poll() == None): self._checkStop(threadName = threadName) time.sleep(float(config['exec_thread_sleep_time'])) if self.__partial: self.processExecResponse(self.__partial) self.logExecOutput(self.__partial) self.log(self.__partial) self.log('cmd(%s) output done' % display_cmd) returnCode = self.__cmdProcess.poll() self.logExecOutput('exitcode: %s' % returnCode) # the error condition if (returnCode != 0): self._updateStatus(httpStatus = 500) # read from stderr and log if self.__cmdProcess.stderr is not None: lines = self.readStream(self.__cmdProcess.stderr) for line in lines: self.processExecResponse(line) if lines: self.logExecOutput('stderr: %s' % '\n'.join(lines)) self.log('cmd(%s) stderr: %s' % (display_cmd, '\n'.join(lines))) self.__LOG.warning(self.__response) errorCode = int(self.__response['error']) if 'error' in self.__response else returnCode msg = 'Application script error (%s) error code (%d) error msg (%s)' % (self.__cmd, errorCode, (self.__response['errorMsg'] if 'errorMsg' in self.__response else '')) self.__LOG.warning(msg) # now add 16000 to the error code to indicate this is a client error clientErrorCode = Errors.CLIENT_SCRIPT_ERROR + abs(errorCode) self._updateStatus(error = clientErrorCode, errorMsg = msg) else: self._updateStatus(httpStatus = 200) if 'result' in self.__response: self._updateStatus(result = self.__response['result']) except SystemExit as excep: # status already set in agent thread msg = "System Exception for cmd %s - %s" % (self.__cmd, self.getStatus()['errorMsg']) self.__LOG.warning("%s - %s" % (msg, str(excep))) raise excep except OSError as excep: if self.__cmdProcess is None: msg = 'Cannot create subprocess cmd(%s) %s - %s' % (self.__cmd, str(excep), traceback.format_exc(2)) else: msg = 'Unknown OSError cmd(%s) %s - %s' % (self.__cmd, str(excep), traceback.format_exc(2)) self.__LOG.warning(msg) self._updateStatus(httpStatus = 500, error = Errors.UNKNOWN_ERROR, errorMsg = msg) except Exception as excep: msg = 'Unknown error cmd(%s) %s - %s' % (self.__cmd, str(excep), traceback.format_exc(2)) self.__LOG.warning(msg) self._updateStatus(httpStatus = 500, error = Errors.UNKNOWN_ERROR, errorMsg = msg) finally: self._updateStatus(progress = 100) if self.__cmdProcess: if (self.__cmdProcess.stdout != None): self.__cmdProcess.stdout.close() if (self.__cmdProcess.stdin != None): self.__cmdProcess.stdin.close() if (self.__cmdProcess.stderr != None): self.__cmdProcess.stderr.close() self.kill(self.__cmdProcess, self.__cmd) if outputLogged: self.logExecOutput('%s uuid end' % (self.getUuid())) self.log("Exec Thread done, cmd=%s" % self.__cmd)
def dataPath(self): """ compute the path to the packages """ datapath = os.path.realpath(os.path.join(manifestutil.servicePath('agent'), '.data')) if not os.path.exists(datapath): os.system('mkdir %s' % datapath) return datapath
def startServicesOnAgentStartup(): """ when agent is restarted, 0. check for agent selfupdate 1. start all service with active manifest, this requires service startup script be idempotent 2. load dynamic controllers and routes if any """ # check for agent update from agent.lib.agenthealth import checkAgentVersion checkAgentVersion(True) # startup services from agent.lib.agent_thread.startstop_service import StartStopService appGlobal = config["pylons.app_globals"] appdelay = int(config["appinitdelay"]) if "appinitdelay" in config else 0 if appdelay > 0: time.sleep(appdelay) # check if this is agent restart or system restart if os.name != "nt" and os.path.exists("/proc/uptime"): uptime, _ = [float(f) for f in open("/proc/uptime").read().split()] else: uptime = 500 systemRestartTimeThreshold = pylons.config["system_restart_time_threshold"] actionType = StartStopService.ACTION_RESTART if int(systemRestartTimeThreshold) > uptime: actionType = StartStopService.ACTION_REBOOT for service in manifestutil.getServices(): appDataDir = manifestutil.appDataPath(service) if not os.path.exists(appDataDir): os.makedirs(appDataDir) import pwd uname = pylons.config["app_user_account"] uid = pwd.getpwnam(uname).pw_uid gid = pwd.getpwnam(uname).pw_gid utils.rchown(appDataDir, uid, gid) dataDir = manifestutil.dataPath(service) if not os.path.exists(dataDir): os.makedirs(dataDir) if service != "agent": try: manifestutil.updateServiceMetaFile( service, {"servicePath": manifestutil.servicePath(service), "serviceName": service} ) except Exception as excep: LOG.error( "Unknown error updating local metadata service(%s) - %s - %s" % (service, str(excep), traceback.format_exc(2)) ) if manifestutil.hasActiveManifest(service): try: LOG.info("startup for service(%s)", service) startupThread = StartStopService(appGlobal.threadMgr, service, actionType) startupThread.start() startupThread.threadMgrEvent.wait() except Exception as excep: LOG.error( "Unknown error starting service(%s) - %s - %s" % (service, str(excep), traceback.format_exc(2)) ) try: LOG.info("load controllers and routes for service(%s)", service) manifestutil.processControllerInPackage(service) except Exception as excep: LOG.error( "Unknown error loading controllers for service(%s) - %s - %s" % (service, str(excep), traceback.format_exc(2)) )