def test_multiple_manifest_create(self): """ when a manifest creation is in progress for a service, another creation should block """ packages = ["http://repository.qa.ebay.com/cronus/test-data/agent/pkgA-1.2.0.unix.cronus"] service = 'foo' manifest1 = 'bar' manifest2 = 'car' try: path = ServiceController.manifestPath(service) os.makedirs(path) path = ServiceController.installedPkgPath(service) os.makedirs(path) except Exception as excep: LOG.warning('got an OS Exception - %s' % str(excep)) body = json.dumps({'package' : packages}) response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest1), headers = {'Content-Type' : 'application/json'}, params = body) self.assertEquals(response1.status_int, 200, 'Manifest1 Post assert - should go through') try: response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest2), headers = {'Content-Type' : 'application/json'}, params = body) self.assertFalse(True, 'Expected an exception but did not get one!') except AppError: pass checkStatus(self, 'create manifest bar', response1, timeout = 25) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'pkgA'))) self.assertFalse(islink(os.path.join(ManifestController.manifestPath('foo', 'car'), 'pkgA')))
def test_post_manifest_inprogress_ok(self): service = 'foo' manifest = 'blahblah' try: path = ServiceController.servicePath(service) if os.path.exists(path): if os.name == 'nt': cmd = 'rm -r %s' % path LOG.debug("running command %s" % cmd) os.system(cmd) else: shutil.rmtree(path) path = ServiceController.manifestPath(service) os.makedirs(path) path = ServiceController.installedPkgPath(service) os.makedirs(path) inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(service, manifest)) os.makedirs(inProgressPath) path = ServiceController.downloadedPkgPath(service) os.makedirs(path) except Exception as excep: LOG.warning('got an OS Exception - %s' % str(excep)) createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus"], manifest = 'blahblah', createDirs = False) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'blahblah'), 'pkgA')))
def __delAllExceptActiveManifests(self, service): ''' Delete all manifests in the given service except active link and corresponding manifest (if one exists). Doesn't delete the corresponding dir in installed-packages (similar to 'delete manifest') ''' manifests = ManifestController.getManifests(service) active = ManifestController.getActiveManifest(service) if active and len(active) != 0: # 'active' can be none manifests.remove(active) for mft in manifests: mf_path = ManifestController.manifestPath(service, mft) ServiceDelete.deleteFolderContents(mf_path)
def test_delete(self): createManifest(self) response = self.app.delete(url(controller = 'manifest', action = 'delete', service = "foo", manifest = "bar"), expect_errors = True) print '*************** = ' + response.body self.assertEquals(200, response.status_int) for _ in range(10): if not os.path.isdir(ManifestController.manifestPath('foo', 'bar')): break else: time.sleep(0.1) self.assertFalse(os.path.isdir(ManifestController.manifestPath('foo', 'bar')))
def test_getAllSymLinks(self): #import pdb; #pdb.set_trace() serviceName = '.sbe.appService.SI1' manifestName = 'manifestA' try: createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"], service = serviceName, manifest = manifestName) except Exception as ex: print 'exception thrown during mf' symLinks = ManifestController.getAllSymLinks(serviceName) """ return all the symlinks from manifests to packages for a given service""" LOG.debug('calling getAllSymLinks') #serviceName = '.sbe.appService.SI1' linkedPaths = [] #pdb.set_trace() #manPath = '/ebay/srengarajan/e2e/local-deploy/service_nodes/.sbe.appService.SI1/manifests' installedPkgPath = os.path.join(ServiceController.servicePath(serviceName), 'installed-packages') pathToPkgA = os.path.join(installedPkgPath, 'pkgA') pathToPkgA120 = os.path.join(pathToPkgA, '1.2.0.unix') pathToPkgB = os.path.join(installedPkgPath, 'pkgB') pathToPkgB060 = os.path.join(pathToPkgB, '0.6.0.unix') bitmap = 0 for path in symLinks: if path.find(pathToPkgA120) >= 0: self.assertTrue(os.path.isdir(path)) bitmap |= 1 elif path.find(pathToPkgB060) >= 0: self.assertTrue(os.path.isdir(path)) bitmap |= 2 self.assertEquals(3, bitmap)
def reset(self, service): ''' Controller to reset service ''' LOG.info('reset for service(%s)', service) try: appGlobal = config['pylons.app_globals'] if not os.path.exists( ManifestController.manifestPath(service, 'active')): return errorResult(request, response, Errors.ACTIVEMANIFEST_MANIFEST_MISSING, 'Active Manifest(%s) path missing' % (service), controller=self) resetThread = ResetService(appGlobal.threadMgr, service) self.injectJobCtx(resetThread) resetThread.start() resetThread.threadMgrEvent.wait() return statusResult(request, response, resetThread, controller=self) except Exception as excep: return errorResult( request, response, error=Errors.UNKNOWN_ERROR, errorMsg='Unknown error for restart service(%s) - %s - %s' % (service, str(excep), traceback.format_exc(2)), controller=self)
def cleanupOrphanedPackages(checkAge = False): ''' API to cleanup Orphaned Packages For All Services ''' from agent.controllers.service import ServiceController #services = os.listdir(service_nodes) from agent.controllers.manifest import ManifestController #import pdb;pdb.set_trace() services = ServiceController.getServices() service_nodes = ServiceController.serviceRootPath() linkedPaths = [] sysServices = ['agent'] LOG.info('Garbage collecting orphaned installed packages') for service in services: try: if service in sysServices: LOG.debug('system services cannot be garbage collected') else: servicePath = os.path.join(service_nodes, service) installedPkgPath = os.path.join(servicePath, 'installed-packages') linkedPaths.extend(ManifestController.getAllSymLinks(service)) linkedPaths.extend(manifestutil.getModuleSymLinks(service)) LOG.debug('symLinks returned %s' % linkedPaths) installedPkgPaths = PackageUtil.getAllInstalledPackages(installedPkgPath) LOG.debug('installedPkgPaths returned for the service %s' % installedPkgPaths) if len(installedPkgPaths) > 0: orphanPkgs = set(installedPkgPaths) - set(linkedPaths) LOG.debug('orphanPkgs returned %s' % orphanPkgs) PackageUtil.cleanupInstalledPkgs(installedPkgPath, orphanPkgs) except BaseException as excep: LOG.error('Failed to proceed with garbage collection %s' % str(excep)) # agent-804, manifests only contains folders, need to delete if file is in manifests servicePath = os.path.join(service_nodes, service) if not os.path.isdir(servicePath): utils.runsyscmd('rm -f %s' % servicePath) LOG.info('Garbage collecting orphaned installed packages completed')
def activatemanifest(self, service): """ activate manifest, if already active then skip """ LOG.info('activateManifest for service(%s) with body: %s', service, request.body) try: appGlobal = config['pylons.app_globals'] manifest = '' requestjson = json.loads(request.body) manifest = requestjson['manifest'] force = asbool(requestjson['force']) if 'force' in requestjson else False if not force and manifestutil.getActiveManifest(service) == manifest: return doneResult(request, response, controller=self) else: mf_path = os.path.join(ManifestController.manifestPath(service, manifest)) if (not os.path.exists(mf_path)): return errorResult(request, response, Errors.ACTIVEMANIFEST_MANIFEST_MISSING, 'Manifest(%s, %s) path missing' % (service, manifest), controller = self) LOG.debug('Manifest path exists: %s' % (mf_path)) activateThread = ActivateManifest(appGlobal.threadMgr, service, manifest) self.injectJobCtx(activateThread) activateThread.start() activateThread.threadMgrEvent.wait() return statusResult(request, response, activateThread, controller = self) except Exception as excep: msg = 'Unknown error for activateManifest(%s/%s) - %s - %s' % (service, manifest, str(excep), traceback.format_exc(2)) return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg = msg, controller = self)
def test_post2(self): # successful post createManifest(self) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix', 'cronus', 'scripts', 'activate'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_package_reuse(self): createManifest(self) body = json.dumps({'package' : ['/packages/perlserver']}) body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"), headers = {'Content-Type' : 'application/json'}, params = body, expect_errors = True) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix', 'cronus', 'scripts', 'activate'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver'))) for _ in range(10): if islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')): break time.sleep(1) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')))
def _obtainCmdExecThreads(self, exeName, service, manifest, activateFlow): """start the threads to execute cmd in each package in the service/manifest""" execThreads = [] packages = ManifestController.getPackages(service, os.path.basename(manifest)) # Executing the deactivate in reversed order where the last package is first deactivated. if (not activateFlow): packages.reverse() # make sure there's something to do, otherwise just return here if (len(packages) == 0): return execThreads for package in packages: pkgPath = manifestutil.packagePath(service, manifest, package) pkgInit = PkgInitConfig(pkgPath) pkgConfig = pkgInit.getConfigs() # check command type and package startup process if (exeName == 'reboot' or exeName == 'restart'): # default action for reboot is to startup, noop for restart agent action = 'startup' if 'reboot' == exeName else 'noop' if exeName == 'reboot' and pkgConfig and pkgConfig.has_key( 'lcm_on_system_reboot'): action = pkgConfig['lcm_on_system_reboot'] if (exeName == 'restart' and pkgConfig and pkgConfig.has_key('lcm_on_agent_restart')): action = pkgConfig['lcm_on_agent_restart'] if action == 'reset': execThread, dummy = self._getBuiltThread( service, manifest, package, "shutdown") execThreads.append([execThread, dummy]) execThread, dummy = self._getBuiltThread( service, manifest, package, "deactivate") execThreads.append([execThread, dummy]) execThread, dummy = self._getBuiltThread( service, manifest, package, "activate") execThreads.append([execThread, dummy]) if (action == 'reset' or action == 'startup'): execThread, dummy = self._getBuiltThread( service, manifest, package, 'startup') execThreads.append([execThread, dummy]) else: execThread, dummy = self._getBuiltThread( service, manifest, package, exeName) execThreads.append([execThread, dummy]) return execThreads
def test_post_without_service(self): body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']}) response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"), headers = {'Content-Type' : 'application/json'}, params = body) assert response.status_int == 200, 'Manifest Post assert' time.sleep(1) self.assertFalse(os.path.exists(ManifestController.manifestPath('foo', 'bar')))
def test_post3_pkg_already_installed(self): createManifest(self) # now lets remove the manifest path path = os.path.join(ManifestController.manifestPath('foo', 'bar')) if os.name == 'nt': cmd = 'rm -r %s' % path LOG.debug("running command %s" % cmd) os.system(cmd) else: shutil.rmtree(path) # now create the manifest again createManifest(self) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix', 'cronus', 'scripts', 'activate'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def testGet(self): path = ManifestController.manifestPath('foo', 'bar') os.makedirs(path) path = ManifestController.manifestPath('foo', 'baz') os.makedirs(path) activePath = os.path.join(ServiceController.manifestPath('foo'), 'active') symlink('bar', activePath) response = self.app.get(url(controller='service', service='foo', action='get'), expect_errors=True) body = json.loads(response.body) print "************** response = %s" % body assert body['progress'] == 100 assert body['result']['activemanifest'] == 'bar' assert body['result']['manifest'] == ['bar', 'baz']
def test_same_pkg_download_parallel(self): packages = ['http://repository.qa.ebay.com/cronus/test-data/agent/perlserver-1.0.0.unix.cronus'] manifest1 = 'bar' manifest2 = 'blah' service1 = 'foo' service2 = 'lah' try: path = ServiceController.manifestPath(service1) os.makedirs(path) path = ServiceController.installedPkgPath(service1) os.makedirs(path) path = ServiceController.downloadedPkgPath(service1) os.makedirs(path) path = ServiceController.manifestPath(service2) os.makedirs(path) path = ServiceController.installedPkgPath(service2) os.makedirs(path) path = ServiceController.downloadedPkgPath(service2) os.makedirs(path) except Exception as excep: LOG.warning('got an OS Exception - %s' % str(excep)) body = json.dumps({'package' : packages}) response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1), headers = {'Content-Type' : 'application/json'}, params = body) assert response1.status_int == 200, 'Manifest Post assert' response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2), headers = {'Content-Type' : 'application/json'}, params = body) assert response2.status_int == 200, 'Manifest Post assert' checkStatus(self, 'create manifest bar', response1, timeout = 25) checkStatus(self, 'create manifest baz', response2, timeout = 25) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('lah', 'blah'), 'perlserver')))
def _getBuiltThread(self, service, manifest, package, exeName): """ here """ # figure out the path to the cronus scripts uname = pylons.config['app_user_account'] execPath = os.path.join(ManifestController.manifestPath(service, manifest), package, 'cronus', 'scripts', exeName) if (isHigherPrivilegeService(service)): cmd = execPath else: cmd = utils.sudoCmd([execPath], uname) dummy = not os.path.exists(execPath) execThread = ExecThread(self._threadMgr, cmd) copycontexts(self, execThread, contextutils.CTX_NAMES) return execThread, dummy
def _obtainCmdExecThreads(self, exeName, service, manifest, activateFlow): """start the threads to execute cmd in each package in the service/manifest""" execThreads = [] packages = ManifestController.getPackages(service, os.path.basename(manifest)) # Executing the deactivate in reversed order where the last package is first deactivated. if (not activateFlow): packages.reverse() # make sure there's something to do, otherwise just return here if (len(packages) == 0): return execThreads for package in packages: pkgPath = manifestutil.packagePath(service, manifest, package) pkgInit = PkgInitConfig(pkgPath) pkgConfig = pkgInit.getConfigs() # check command type and package startup process if (exeName == 'reboot' or exeName == 'restart'): # default action for reboot is to startup, noop for restart agent action = 'startup' if 'reboot' == exeName else 'noop' if exeName == 'reboot' and pkgConfig and pkgConfig.has_key('lcm_on_system_reboot'): action = pkgConfig['lcm_on_system_reboot'] if (exeName == 'restart' and pkgConfig and pkgConfig.has_key('lcm_on_agent_restart')): action = pkgConfig['lcm_on_agent_restart'] if action == 'reset': execThread, dummy = self._getBuiltThread(service, manifest, package, "shutdown") execThreads.append([execThread, dummy]) execThread, dummy = self._getBuiltThread(service, manifest, package, "deactivate") execThreads.append([execThread, dummy]) execThread, dummy = self._getBuiltThread(service, manifest, package, "activate") execThreads.append([execThread, dummy]) if (action == 'reset' or action == 'startup'): execThread, dummy = self._getBuiltThread(service, manifest, package, 'startup') execThreads.append([execThread, dummy]) else: execThread, dummy = self._getBuiltThread(service, manifest, package, exeName) execThreads.append([execThread, dummy]) return execThreads
def _getBuiltThread(self, service, manifest, package, exeName): """ here """ # figure out the path to the cronus scripts uname = pylons.config['app_user_account'] execPath = os.path.join( ManifestController.manifestPath(service, manifest), package, 'cronus', 'scripts', exeName) if (isHigherPrivilegeService(service)): cmd = execPath else: cmd = utils.sudoCmd([execPath], uname) dummy = not os.path.exists(execPath) execThread = ExecThread(self._threadMgr, cmd) copycontexts(self, execThread, contextutils.CTX_NAMES) return execThread, dummy
def test_post3_already_installed_manifest(self): createManifest(self) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix', 'cronus', 'scripts', 'activate'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver'))) body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']}) response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"), headers = {'Content-Type' : 'application/json'}, params = body, expect_errors = True) self.assertEquals(201, response.status_int, 'Manifest Post assert') body = json.loads(response.body) assert response.status_int == 201, 'Manifest Post assert'
def activatemanifest(self, service): """ activate manifest, if already active then skip """ LOG.info('activateManifest for service(%s) with body: %s', service, request.body) try: appGlobal = config['pylons.app_globals'] manifest = '' requestjson = json.loads(request.body) manifest = requestjson['manifest'] force = asbool( requestjson['force']) if 'force' in requestjson else False if not force and manifestutil.getActiveManifest( service) == manifest: return doneResult(request, response, controller=self) else: mf_path = os.path.join( ManifestController.manifestPath(service, manifest)) if (not os.path.exists(mf_path)): return errorResult(request, response, Errors.ACTIVEMANIFEST_MANIFEST_MISSING, 'Manifest(%s, %s) path missing' % (service, manifest), controller=self) LOG.debug('Manifest path exists: %s' % (mf_path)) activateThread = ActivateManifest(appGlobal.threadMgr, service, manifest) self.injectJobCtx(activateThread) activateThread.start() activateThread.threadMgrEvent.wait() return statusResult(request, response, activateThread, controller=self) except Exception as excep: msg = 'Unknown error for activateManifest(%s/%s) - %s - %s' % ( service, manifest, str(excep), traceback.format_exc(2)) return errorResult(request, response, error=Errors.UNKNOWN_ERROR, errorMsg=msg, controller=self)
def test_inprogress_pkg_download(self): service = 'foo' try: path = ServiceController.manifestPath(service) os.makedirs(path) path = ServiceController.installedPkgPath(service) os.makedirs(path) path = ServiceController.downloadedPkgPath(service) os.makedirs(path) inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress') inprogressFile = open(inprogressPath, 'w') inprogressFile.write('somegarbage') inprogressFile.close() except Exception as excep: LOG.warning('got an OS Exception - %s' % str(excep)) createManifest(self) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def cleanupOrphanedPackages(checkAge=False): ''' API to cleanup Orphaned Packages For All Services ''' from agent.controllers.service import ServiceController #services = os.listdir(service_nodes) from agent.controllers.manifest import ManifestController #import pdb;pdb.set_trace() services = ServiceController.getServices() service_nodes = ServiceController.serviceRootPath() linkedPaths = [] sysServices = ['agent'] LOG.info('Garbage collecting orphaned installed packages') for service in services: try: if service in sysServices: LOG.debug('system services cannot be garbage collected') else: servicePath = os.path.join(service_nodes, service) installedPkgPath = os.path.join(servicePath, 'installed-packages') linkedPaths.extend( ManifestController.getAllSymLinks(service)) linkedPaths.extend(manifestutil.getModuleSymLinks(service)) LOG.debug('symLinks returned %s' % linkedPaths) installedPkgPaths = PackageUtil.getAllInstalledPackages( installedPkgPath) LOG.debug('installedPkgPaths returned for the service %s' % installedPkgPaths) if len(installedPkgPaths) > 0: orphanPkgs = set(installedPkgPaths) - set(linkedPaths) LOG.debug('orphanPkgs returned %s' % orphanPkgs) PackageUtil.cleanupInstalledPkgs( installedPkgPath, orphanPkgs) except BaseException as excep: LOG.error('Failed to proceed with garbage collection %s' % str(excep)) # agent-804, manifests only contains folders, need to delete if file is in manifests servicePath = os.path.join(service_nodes, service) if not os.path.isdir(servicePath): utils.runsyscmd('rm -f %s' % servicePath) LOG.info('Garbage collecting orphaned installed packages completed')
def test_getPackagePaths(self): serviceName = 'service1' manifestName = 'manifestA' createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"], service = serviceName, manifest = manifestName) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus'))) packagePaths = ManifestController.getPackagePaths(serviceName, manifestName); print ">>>>>>>>>>>>" + str(packagePaths) self.assertTrue(len(packagePaths) >= 2) bitmap = 0 for path in packagePaths: if path.find("pkgA") >= 0: self.assertTrue(os.path.isdir(path)) bitmap |= 1 elif path.find("pkgB") >= 0: self.assertTrue(os.path.isdir(path)) bitmap |= 2 self.assertEquals(3, bitmap)
def reset(self, service): ''' Controller to reset service ''' LOG.info('reset for service(%s)', service) try: appGlobal = config['pylons.app_globals'] if not os.path.exists(ManifestController.manifestPath(service, 'active')): return errorResult(request, response, Errors.ACTIVEMANIFEST_MANIFEST_MISSING, 'Active Manifest(%s) path missing' % (service), controller = self) resetThread = ResetService(appGlobal.threadMgr, service) self.injectJobCtx(resetThread) resetThread.start() resetThread.threadMgrEvent.wait() return statusResult(request, response, resetThread, controller = self) except Exception as excep: return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg = 'Unknown error for restart service(%s) - %s - %s' % (service, str(excep), traceback.format_exc(2)), controller = self)
def testDelete(self): def deleteTestDir(service): LOG.debug('************ service = %s' % service) response = self.app.delete(url(controller='service', service=service, action='delete'), expect_errors = True) # make sure the responses are correct LOG.debug('status = ' + str(response.status_int)) # assert response.status_int == 500, "HTTP response != 500" def makeTestDir(path): os.makedirs(path) os.makedirs(os.path.join(path, 'manifests')) os.makedirs(os.path.join(path, 'installed-packages')) def createManifests(mf_path): os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0')) os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0')) latest = os.path.join(mf_path, 'm3.0') os.makedirs(os.path.join(latest, 'dummy_dir3.0')) utils.symlink(latest, os.path.join(mf_path, 'active')) return (['m1.0', 'm2.0', 'm3.0'], 'm3.0') def makePackageContent(path, pkgPath, pkgPropPath): pkgFile = file(pkgPath, 'w') for index in range(10): pkgFile.write(('%s%s') % (index, index)) pkgFile.close() pkgFile = file(pkgPropPath, 'w') for index in range(10): pkgFile.write(('%s%s') % (index, index)) pkgFile.close() uname = pylons.config['agent_user_account'] TestCleanupController.rchown(path, uname) def createTestThread(serviceName): appGlobal = config['pylons.app_globals'] testTh = WaitThread(appGlobal.threadMgr, ServiceController.serviceCat(serviceName)) testTh.start() return testTh def startTestProcess(): cmd = utils.sudoCmd(["sleep", "5"], pylons.config['app_user_account']) return Popen(cmd) if os.name == 'nt': LOG.warning('Services cleanup not supported on windows. Skipping test...') return path1 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'foo') path2 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'bar') path3 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'agent') deleteTestDir('foo') deleteTestDir('bar') deleteTestDir('agent') # make dirs makeTestDir(path1) makeTestDir(path2) makeTestDir(path3) all_mf, active_mf = createManifests(ServiceController.manifestPath('agent')) uname = pylons.config['agent_user_account'] TestCleanupController.rchown(ServiceController.serviceRootPath(), uname) pkgDir = PackageMgr.packagePath() pkgPath = os.path.join(pkgDir, "foo.cronus") pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop") makePackageContent(pkgDir, pkgPath, pkgPropPath) # create threads testThFoo = createTestThread('foo') testThBar = createTestThread('bar') testThAgent = createTestThread('agent') # start process process = startTestProcess() # start testing LOG.debug('************ start cleanup') response = self.app.post(url(controller='cleanup', action='post')) LOG.debug ('Delete response body = ' + response.body) body = json.loads(response.body) tm = time.time() while (tm + 10 > time.time()): response = self.app.get(body['status'], expect_errors = True) LOG.debug ('Status response body = ' + response.body) body = json.loads(response.body) print body if (body['progress'] == 100): break time.sleep(0.1) # make sure the responses are correct LOG.debug('status = ' + str(response.status_int)) assert response.status_int == 200, "HTTP response != 200" time.sleep(0.1) assert not os.path.exists(path1), 'service foo does exist or is not a directory' assert not os.path.exists(path2), 'service bar does exist or is not a directory' assert os.path.exists(path3), 'service agent does NOT exist or is not a directory' assert not testThFoo.isAlive(), 'thread Foo is still alive' assert not testThBar.isAlive(), 'thread Bar is still alive' assert not testThAgent.isAlive(), 'thread Agent is still alive' assert not os.path.exists(pkgPath), 'package foo exists' assert not os.path.exists(pkgPropPath), 'package prop foo exists' assert os.path.exists(pkgDir), 'package directory does not exist' # ensure agent cleanup is proper active_mf_path = ManifestController.manifestPath('agent', active_mf) active_link = os.path.join(ServiceController.manifestPath('agent'), 'active') all_mf.remove(active_mf) actual_active_mf_path = utils.readlink(active_link) self.assertTrue(os.path.exists(active_mf_path), 'active agent manifest got deleted but shouldn\t have') self.assertTrue(os.path.exists(active_link), 'agent active link missing') self.assertEqual(active_mf_path, actual_active_mf_path, 'agent active link pointing to some wrong manifest; link broken?') for mf in all_mf: agnt_mf_path = ManifestController.manifestPath('agent', mf) self.assertFalse(os.path.exists(agnt_mf_path), 'non active agent mf %s should have been deleted' % mf)
def doRun(self): """ Main body of the thread Progress Info: Look at the list of packages this manifest has and figure out the progress of this manifest Progress 1 = manifest directory created Progress 2-80 = all packages downloaded Progress 81-99 = all packages untarred Progress 100 = all links done """ inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(self.__service, self.__manifest)) try: if self.__service != 'agent': utils.checkDiskFull() installedPkgPath = ServiceController.installedPkgPath(self.__service) # This shouldn'trn happen but make sure there isn'trn already a manifest directory in progress if (os.path.isdir(inProgressPath)): # probably another manifest create thread died out half way. # Cleanup and reattempt manifest creation LOG.debug('Manifest inprogress found for service/manifest (%s/%s). Will cleanup and retry' % (self.__service, self.__manifest)) shutil.rmtree(inProgressPath) # make sure that the service path and installed package path exists if (not os.path.isdir(ServiceController.manifestPath(self.__service)) or not os.path.isdir(ServiceController.installedPkgPath(self.__service))): errCode = Errors.SERVICE_NOT_FOUND msg = 'Service (%s) for manifest (%s) not found' % (self.__service, self.__manifest) self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg) return # ok create the manifest path os.mkdir(inProgressPath) self._updateProgress(1) # figure out which of the packages are already there remainingPackages = {} for pkgUri in self.__packages: pkgDict = PackageUtil.parseUri(pkgUri) pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion']) pkgName = pkgDict['packageName'] if (not os.path.exists(pkgPath)) or ((self.__forcePackages is not None) and pkgName in self.__forcePackages): remainingPackages[pkgUri] = pkgDict else: symlink(pkgPath, os.path.join(inProgressPath, pkgDict['packageName'])) if self.__attemptDownload: # now make sure all the packages are downloaded try: self._downloadPackages(remainingPackages.keys(), skipProp = self.__skipProp) except AgentException as exc: # check if it is download error, then modify exception appropriately if exc.getCode() == Errors.DC_FAILED_DOWNLOAD: exc = AgentException(Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED, 'Manifest (%s/%s) failed downloading package - %s' % (self.__service, self.__manifest, exc.getMsg())) raise exc else: if len(remainingPackages) > 0: raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST, 'Create Manifest (%s/%s) failed since package is not present and download has been disabled' % (self.__service, self.__manifest)) LOG.info('Completed download all packages for (%s/%s)' % (self.__service, self.__manifest)) # now untar the packages import re pkgSuffix = '.%s' % re.sub(r"\W", "", self.__manifest) self._untarPackages(remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service), 0, self.__forcePackages, pkgSuffix) LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__manifest)) # now create the links for pkgDict in remainingPackages.itervalues(): pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion']) linkPath = os.path.join(inProgressPath, pkgDict['packageName']) # validate target folder does exist if not os.path.exists(pkgPath): raise AgentException(Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath) symlink(pkgPath, linkPath) # now move the inProgressPath to the final path manifestContentPath = ManifestController.manifestContentPath(self.__service, self.__manifest) os.rename(inProgressPath, ManifestController.manifestPath(self.__service, self.__manifest)) mfContentsFile = file(manifestContentPath, 'w') for pkgUri in self.__packages: mfContentsFile.write(('%s%s') % (pkgUri, os.linesep)) mfContentsFile.close() LOG.info('Completed create manifest directories for (%s/%s)' % (self.__service, self.__manifest)) LOG.info('Completed create manifest for (%s/%s)' % (self.__service, self.__manifest)) self._updateStatus(progress = 100) except AgentException as exc: LOG.info(exc.getMsg()) self._updateStatus(httpStatus = 500, error = exc.getCode(), errorMsg = exc.getMsg()) except Exception as exc: errCode = Errors.UNKNOWN_ERROR msg = 'Unknown error for (%s/%s) - %s - %s' % (self.__service, self.__manifest, str(exc), traceback.format_exc(2)) LOG.info(msg) self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg) finally: # clean up intermediate progress try: shutil.rmtree(inProgressPath) except OSError: pass
def testDelete(self): def deleteTestDir(service): LOG.debug('************ service = %s' % service) response = self.app.delete(url(controller='service', service=service, action='delete'), expect_errors=True) # make sure the responses are correct LOG.debug('status = ' + str(response.status_int)) # assert response.status_int == 500, "HTTP response != 500" def makeTestDir(path): os.makedirs(path) os.makedirs(os.path.join(path, 'manifests')) os.makedirs(os.path.join(path, 'installed-packages')) def createManifests(mf_path): os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0')) os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0')) latest = os.path.join(mf_path, 'm3.0') os.makedirs(os.path.join(latest, 'dummy_dir3.0')) utils.symlink(latest, os.path.join(mf_path, 'active')) return (['m1.0', 'm2.0', 'm3.0'], 'm3.0') def makePackageContent(path, pkgPath, pkgPropPath): pkgFile = file(pkgPath, 'w') for index in range(10): pkgFile.write(('%s%s') % (index, index)) pkgFile.close() pkgFile = file(pkgPropPath, 'w') for index in range(10): pkgFile.write(('%s%s') % (index, index)) pkgFile.close() uname = pylons.config['agent_user_account'] TestCleanupController.rchown(path, uname) def createTestThread(serviceName): appGlobal = config['pylons.app_globals'] testTh = WaitThread(appGlobal.threadMgr, ServiceController.serviceCat(serviceName)) testTh.start() return testTh def startTestProcess(): cmd = utils.sudoCmd(["sleep", "5"], pylons.config['app_user_account']) return Popen(cmd) if os.name == 'nt': LOG.warning( 'Services cleanup not supported on windows. Skipping test...') return path1 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'foo') path2 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'bar') path3 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'agent') deleteTestDir('foo') deleteTestDir('bar') deleteTestDir('agent') # make dirs makeTestDir(path1) makeTestDir(path2) makeTestDir(path3) all_mf, active_mf = createManifests( ServiceController.manifestPath('agent')) uname = pylons.config['agent_user_account'] TestCleanupController.rchown(ServiceController.serviceRootPath(), uname) pkgDir = PackageMgr.packagePath() pkgPath = os.path.join(pkgDir, "foo.cronus") pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop") makePackageContent(pkgDir, pkgPath, pkgPropPath) # create threads testThFoo = createTestThread('foo') testThBar = createTestThread('bar') testThAgent = createTestThread('agent') # start process process = startTestProcess() # start testing LOG.debug('************ start cleanup') response = self.app.post(url(controller='cleanup', action='post')) LOG.debug('Delete response body = ' + response.body) body = json.loads(response.body) tm = time.time() while (tm + 10 > time.time()): response = self.app.get(body['status'], expect_errors=True) LOG.debug('Status response body = ' + response.body) body = json.loads(response.body) print body if (body['progress'] == 100): break time.sleep(0.1) # make sure the responses are correct LOG.debug('status = ' + str(response.status_int)) assert response.status_int == 200, "HTTP response != 200" time.sleep(0.1) assert not os.path.exists( path1), 'service foo does exist or is not a directory' assert not os.path.exists( path2), 'service bar does exist or is not a directory' assert os.path.exists( path3), 'service agent does NOT exist or is not a directory' assert not testThFoo.isAlive(), 'thread Foo is still alive' assert not testThBar.isAlive(), 'thread Bar is still alive' assert not testThAgent.isAlive(), 'thread Agent is still alive' assert not os.path.exists(pkgPath), 'package foo exists' assert not os.path.exists(pkgPropPath), 'package prop foo exists' assert os.path.exists(pkgDir), 'package directory does not exist' # ensure agent cleanup is proper active_mf_path = ManifestController.manifestPath('agent', active_mf) active_link = os.path.join(ServiceController.manifestPath('agent'), 'active') all_mf.remove(active_mf) actual_active_mf_path = utils.readlink(active_link) self.assertTrue( os.path.exists(active_mf_path), 'active agent manifest got deleted but shouldn\t have') self.assertTrue(os.path.exists(active_link), 'agent active link missing') self.assertEqual( active_mf_path, actual_active_mf_path, 'agent active link pointing to some wrong manifest; link broken?') for mf in all_mf: agnt_mf_path = ManifestController.manifestPath('agent', mf) self.assertFalse( os.path.exists(agnt_mf_path), 'non active agent mf %s should have been deleted' % mf)
def _addMonitors(self, service, activeManifest): ''' add monitors for a service's activeManifest ''' monitorTasks = [] self.__monitorTasks[(service, activeManifest)] = monitorTasks # metric tags metricTags = {} # add default metric tags metricTags['fqdn'] = utils.fqdn # read tags from agent agtMetricTags = manifestutil.readJsonServiceMeta('agent', [METRIC_TAGS_KEY]) if agtMetricTags and METRIC_TAGS_KEY in agtMetricTags: metricTags.update(agtMetricTags[METRIC_TAGS_KEY]) # read tags from service if (service != 'agent'): serviceMetricTags = manifestutil.readJsonServiceMeta(service, [METRIC_TAGS_KEY]) if serviceMetricTags or METRIC_TAGS_KEY in serviceMetricTags: metricTags.update(serviceMetricTags[METRIC_TAGS_KEY]) # all packages in this manifest packages = ManifestController.getPackages(service, os.path.basename(activeManifest)) # metric tags for externally passed in metrics # monitorTags = {additionaltags_dict, datatype_dict, ctx(env.pool.host), appType(Monitor|custom), monitorType(Cronus|CronusApplication)} self.__monitorTags[service] = {'default': (metricTags, {}, {})} self.__monitorValues[(service, 'default')] = {} self.__monitorMessages[(service, 'default')] = {} # a unique index for all monitors in the service for package in packages: # load monitor setting from cronus.ini pkgConfigs = manifestutil.getPackageInitConfig(service, activeManifest, package) if pkgConfigs is not None and 'monitors' in pkgConfigs: metricCtx = pkgConfigs[METRIC_CONTEXT] if METRIC_CONTEXT in pkgConfigs else {} metricTypes = {} if METRIC_TYPE_KEY in pkgConfigs and pkgConfigs[METRIC_TYPE_KEY]: metricTypes = pkgConfigs[METRIC_TYPE_KEY] if METRIC_TAGS_KEY in pkgConfigs and pkgConfigs[METRIC_TAGS_KEY]: metricTags.update(pkgConfigs[METRIC_TAGS_KEY]) monitors = pkgConfigs['monitors'] for monitor in monitors: try: m_name = str(monitor['name']) # metric tags for externally passed in metrics # monitorTags = {additionaltags_dict, datatype_dict, ctx(env.pool.host), appType(Monitor|custom), monitorType(Cronus|CronusApplication)} self.__monitorTags[service][m_name] = (metricTags, metricTypes, metricCtx) # now normalize the interval to one allowable by EVPS (downcast) if 'reportIntervalSec' not in monitor: continue m_interval = float(monitor['reportIntervalSec']) m_interval = self._normalizeResolution(m_interval) m_timeout = float(monitor['timeoutSec']) if 'timeoutSec' in monitor else m_interval m_timeout = max(1, min(m_interval-1, m_timeout)) m_items = monitor['items'] if 'items' in monitor else [] # prime monitor value store self.__monitorValues[(service, m_name)] = {} self.__monitorMessages[(service, m_name)] = {} mi_idx = 0 for m_item in m_items: if 'type' not in m_item: continue mi_idx += 1 mi_type = m_item['type'] mi_cmd = None if 'script' == mi_type: mi_script = m_item['script'] mi_cmd = manifestutil.getMonitorScriptPath(service, activeManifest, package, mi_script) else: # unknown type, skip LOG.error('Unknown monitor item type %s, can only be script' % mi_type) continue # schedule monitor m_sch_name = '%s.%s' % (service, m_name) try: self.__monitorMessages[(service, m_name)][mi_cmd] = [] task = self.__monitorSch.add_interval_task(self._runMonitor, '%s.%s' % (m_sch_name, str(mi_idx)), 0, float(m_interval), [mi_type,mi_cmd,service,m_name,m_timeout,(str(int(round(m_interval))),metricTags,metricTypes,metricCtx)],None) monitorTasks.append(task) except Exception as excep: LOG.error('Cannot add monitor task for %s.%s - (%s) %s' % (m_sch_name, str(mi_idx), excep, traceback.format_exc(5))) except Exception as excep1: LOG.error('Cannot add report task for %s.%s - (%s) %s' % (service, m_name, excep1, traceback.format_exc(5)))