def test_cleanupOrphanedPackages2(self): from agent.lib.utils import rchown from agent.controllers.service import ServiceController from agent.controllers.manifest import ManifestController from agent.tests.unit.test_util import createManifest if os.name == 'nt': pass else: serviceName = 'service2' manifestName = 'manifestA' serviceName1 = 'service3' try: createManifest( self, packages=[ "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus" ], service=serviceName, manifest=manifestName) createManifest( self, packages=[ "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus" ], service=serviceName1, manifest=manifestName) except Exception as ex: print 'exception thrown during mf %s' % str(ex) installedPkgPath = os.path.realpath( os.path.join(ServiceController.serviceRootPath(), serviceName, 'installed-packages')) installedPkgPath1 = os.path.realpath( os.path.join(ServiceController.serviceRootPath(), serviceName1, 'installed-packages')) if (not os.name == 'nt'): import pwd uname = pylons.config['agent_user_account'] uid = pwd.getpwnam(uname).pw_uid gid = pwd.getpwnam(uname).pw_gid rchown(installedPkgPath, uid, gid) rchown(installedPkgPath1, uid, gid) self.assertEquals(len(os.listdir(installedPkgPath)), 1) self.assertEquals(len(os.listdir(installedPkgPath1)), 1) self.app.delete(url(controller='manifest', action='delete', service=serviceName, manifest=manifestName), expect_errors=True) self.app.delete(url(controller='manifest', action='delete', service=serviceName1, manifest=manifestName), expect_errors=True) pylons.config['packageMgr_install_package_age'] = 0 pylons.config['packageMgr_install_package_min_age'] = 0.0 PackageUtil.cleanupOrphanedPackages() self.assertEquals(len(os.listdir(installedPkgPath)), 0) self.assertEquals(len(os.listdir(installedPkgPath1)), 0)
def cleanupOrphanedPackages(checkAge = False): ''' API to cleanup Orphaned Packages For All Services ''' from agent.controllers.service import ServiceController #services = os.listdir(service_nodes) from agent.controllers.manifest import ManifestController #import pdb;pdb.set_trace() services = ServiceController.getServices() service_nodes = ServiceController.serviceRootPath() linkedPaths = [] sysServices = ['agent'] LOG.info('Garbage collecting orphaned installed packages') for service in services: try: if service in sysServices: LOG.debug('system services cannot be garbage collected') else: servicePath = os.path.join(service_nodes, service) installedPkgPath = os.path.join(servicePath, 'installed-packages') linkedPaths.extend(ManifestController.getAllSymLinks(service)) linkedPaths.extend(manifestutil.getModuleSymLinks(service)) LOG.debug('symLinks returned %s' % linkedPaths) installedPkgPaths = PackageUtil.getAllInstalledPackages(installedPkgPath) LOG.debug('installedPkgPaths returned for the service %s' % installedPkgPaths) if len(installedPkgPaths) > 0: orphanPkgs = set(installedPkgPaths) - set(linkedPaths) LOG.debug('orphanPkgs returned %s' % orphanPkgs) PackageUtil.cleanupInstalledPkgs(installedPkgPath, orphanPkgs) except BaseException as excep: LOG.error('Failed to proceed with garbage collection %s' % str(excep)) # agent-804, manifests only contains folders, need to delete if file is in manifests servicePath = os.path.join(service_nodes, service) if not os.path.isdir(servicePath): utils.runsyscmd('rm -f %s' % servicePath) LOG.info('Garbage collecting orphaned installed packages completed')
def test_cleanupOrphanedPackages(self): from agent.lib.utils import rchown from agent.controllers.service import ServiceController from agent.controllers.manifest import ManifestController from agent.tests.unit.test_util import createManifest if os.name == 'nt': pass else: serviceName = 'service2' manifestName = 'manifestA' try: createManifest( self, packages=[ "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus" ], service=serviceName, manifest=manifestName) except Exception as ex: print 'exception thrown during mf %s' % str(ex) installedPkgPath = os.path.realpath( os.path.join(ServiceController.serviceRootPath(), serviceName, 'installed-packages')) if (not os.name == 'nt'): import pwd uname = pylons.config['agent_user_account'] uid = pwd.getpwnam(uname).pw_uid gid = pwd.getpwnam(uname).pw_gid rchown(installedPkgPath, uid, gid) #installedPkgPath = os.path.realpath(os.path.join('/ebay/srengarajan/e2e/local-deploy/service_nodes/' , '.sbe.appService.SI1', 'installed-packages' )) #pkgs = os.listdir('C:\\srini\\projects\\gc\\agent\\trunk') self.assertEquals(len(os.listdir(installedPkgPath)), 1) self.app.delete(url(controller='manifest', action='delete', service=serviceName, manifest=manifestName), expect_errors=True) pylons.config['packageMgr_install_package_age'] = 0 pylons.config['packageMgr_install_package_min_age'] = 0.0 time.sleep(1) PackageUtil.cleanupOrphanedPackages() self.assertEquals(len(os.listdir(installedPkgPath)), 0)
def cleanupOrphanedPackages(checkAge=False): ''' API to cleanup Orphaned Packages For All Services ''' from agent.controllers.service import ServiceController #services = os.listdir(service_nodes) from agent.controllers.manifest import ManifestController #import pdb;pdb.set_trace() services = ServiceController.getServices() service_nodes = ServiceController.serviceRootPath() linkedPaths = [] sysServices = ['agent'] LOG.info('Garbage collecting orphaned installed packages') for service in services: try: if service in sysServices: LOG.debug('system services cannot be garbage collected') else: servicePath = os.path.join(service_nodes, service) installedPkgPath = os.path.join(servicePath, 'installed-packages') linkedPaths.extend( ManifestController.getAllSymLinks(service)) linkedPaths.extend(manifestutil.getModuleSymLinks(service)) LOG.debug('symLinks returned %s' % linkedPaths) installedPkgPaths = PackageUtil.getAllInstalledPackages( installedPkgPath) LOG.debug('installedPkgPaths returned for the service %s' % installedPkgPaths) if len(installedPkgPaths) > 0: orphanPkgs = set(installedPkgPaths) - set(linkedPaths) LOG.debug('orphanPkgs returned %s' % orphanPkgs) PackageUtil.cleanupInstalledPkgs( installedPkgPath, orphanPkgs) except BaseException as excep: LOG.error('Failed to proceed with garbage collection %s' % str(excep)) # agent-804, manifests only contains folders, need to delete if file is in manifests servicePath = os.path.join(service_nodes, service) if not os.path.isdir(servicePath): utils.runsyscmd('rm -f %s' % servicePath) LOG.info('Garbage collecting orphaned installed packages completed')
def testDelete(self): def deleteTestDir(service): LOG.debug('************ service = %s' % service) response = self.app.delete(url(controller='service', service=service, action='delete'), expect_errors=True) # make sure the responses are correct LOG.debug('status = ' + str(response.status_int)) # assert response.status_int == 500, "HTTP response != 500" def makeTestDir(path): os.makedirs(path) os.makedirs(os.path.join(path, 'manifests')) os.makedirs(os.path.join(path, 'installed-packages')) def createManifests(mf_path): os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0')) os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0')) latest = os.path.join(mf_path, 'm3.0') os.makedirs(os.path.join(latest, 'dummy_dir3.0')) utils.symlink(latest, os.path.join(mf_path, 'active')) return (['m1.0', 'm2.0', 'm3.0'], 'm3.0') def makePackageContent(path, pkgPath, pkgPropPath): pkgFile = file(pkgPath, 'w') for index in range(10): pkgFile.write(('%s%s') % (index, index)) pkgFile.close() pkgFile = file(pkgPropPath, 'w') for index in range(10): pkgFile.write(('%s%s') % (index, index)) pkgFile.close() uname = pylons.config['agent_user_account'] TestCleanupController.rchown(path, uname) def createTestThread(serviceName): appGlobal = config['pylons.app_globals'] testTh = WaitThread(appGlobal.threadMgr, ServiceController.serviceCat(serviceName)) testTh.start() return testTh def startTestProcess(): cmd = utils.sudoCmd(["sleep", "5"], pylons.config['app_user_account']) return Popen(cmd) if os.name == 'nt': LOG.warning( 'Services cleanup not supported on windows. Skipping test...') return path1 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'foo') path2 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'bar') path3 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'agent') deleteTestDir('foo') deleteTestDir('bar') deleteTestDir('agent') # make dirs makeTestDir(path1) makeTestDir(path2) makeTestDir(path3) all_mf, active_mf = createManifests( ServiceController.manifestPath('agent')) uname = pylons.config['agent_user_account'] TestCleanupController.rchown(ServiceController.serviceRootPath(), uname) pkgDir = PackageMgr.packagePath() pkgPath = os.path.join(pkgDir, "foo.cronus") pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop") makePackageContent(pkgDir, pkgPath, pkgPropPath) # create threads testThFoo = createTestThread('foo') testThBar = createTestThread('bar') testThAgent = createTestThread('agent') # start process process = startTestProcess() # start testing LOG.debug('************ start cleanup') response = self.app.post(url(controller='cleanup', action='post')) LOG.debug('Delete response body = ' + response.body) body = json.loads(response.body) tm = time.time() while (tm + 10 > time.time()): response = self.app.get(body['status'], expect_errors=True) LOG.debug('Status response body = ' + response.body) body = json.loads(response.body) print body if (body['progress'] == 100): break time.sleep(0.1) # make sure the responses are correct LOG.debug('status = ' + str(response.status_int)) assert response.status_int == 200, "HTTP response != 200" time.sleep(0.1) assert not os.path.exists( path1), 'service foo does exist or is not a directory' assert not os.path.exists( path2), 'service bar does exist or is not a directory' assert os.path.exists( path3), 'service agent does NOT exist or is not a directory' assert not testThFoo.isAlive(), 'thread Foo is still alive' assert not testThBar.isAlive(), 'thread Bar is still alive' assert not testThAgent.isAlive(), 'thread Agent is still alive' assert not os.path.exists(pkgPath), 'package foo exists' assert not os.path.exists(pkgPropPath), 'package prop foo exists' assert os.path.exists(pkgDir), 'package directory does not exist' # ensure agent cleanup is proper active_mf_path = ManifestController.manifestPath('agent', active_mf) active_link = os.path.join(ServiceController.manifestPath('agent'), 'active') all_mf.remove(active_mf) actual_active_mf_path = utils.readlink(active_link) self.assertTrue( os.path.exists(active_mf_path), 'active agent manifest got deleted but shouldn\t have') self.assertTrue(os.path.exists(active_link), 'agent active link missing') self.assertEqual( active_mf_path, actual_active_mf_path, 'agent active link pointing to some wrong manifest; link broken?') for mf in all_mf: agnt_mf_path = ManifestController.manifestPath('agent', mf) self.assertFalse( os.path.exists(agnt_mf_path), 'non active agent mf %s should have been deleted' % mf)
def test_cleanUpAllInstalledPackages(self): from agent.lib.utils import rchown from agent.controllers.service import ServiceController from agent.controllers.manifest import ManifestController from agent.tests.unit.test_util import createManifest #import pdb;pdb.set_trace(); serviceName = 'service3' manifestName = 'manifestB' try: createManifest( self, packages=[ "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus" ], service=serviceName, manifest=manifestName) except Exception as ex: print 'exception thrown during mf %s' % str(ex) installedPkgPath = os.path.realpath( os.path.join(ServiceController.serviceRootPath(), serviceName, 'installed-packages')) if (not os.name == 'nt'): import pwd uname = pylons.config['agent_user_account'] uid = pwd.getpwnam(uname).pw_uid gid = pwd.getpwnam(uname).pw_gid rchown(installedPkgPath, uid, gid) #installedPkgPath = os.path.realpath(os.path.join('/ebay/srengarajan/e2e/local-deploy/service_nodes/' , '.sbe.appService.SI1', 'installed-packages' )) #pkgs = os.listdir('C:\\srini\\projects\\gc\\agent\\trunk') pkgs = [] for pkg in os.listdir(installedPkgPath): pkgs.append(os.path.join(installedPkgPath, pkg)) pkgVers = [] for pkg in pkgs: for pkgVer in os.listdir(pkg): pkgVers.append(os.path.join(pkg, pkgVer)) self.assertEquals(len(os.listdir(installedPkgPath)), 1) self.app.delete(url(controller='manifest', action='delete', service=serviceName, manifest=manifestName), expect_errors=True) #import pdb; pdb.set_trace(); symLinks = [] #ManifestController.getAllSymLinks(serviceName) orphans = set(pkgVers) - set(symLinks) age = pylons.config['packageMgr_install_package_age'] minage = pylons.config['packageMgr_install_package_min_age'] pylons.config['packageMgr_install_package_age'] = 0 pylons.config['packageMgr_install_package_min_age'] = 0.0 self.assertEquals(len(os.listdir(installedPkgPath)), 1) PackageUtil.cleanupInstalledPkgs(installedPkgPath, orphans) #PackageUtil.cleanUpInstalledPkgs(installedPkgPath, orphans) self.assertEquals(len(os.listdir(installedPkgPath)), 0) pylons.config['packageMgr_install_package_age'] = age pylons.config['packageMgr_install_package_min_age'] = minage
def testDelete(self): def deleteTestDir(service): LOG.debug('************ service = %s' % service) response = self.app.delete(url(controller='service', service=service, action='delete'), expect_errors = True) # make sure the responses are correct LOG.debug('status = ' + str(response.status_int)) # assert response.status_int == 500, "HTTP response != 500" def makeTestDir(path): os.makedirs(path) os.makedirs(os.path.join(path, 'manifests')) os.makedirs(os.path.join(path, 'installed-packages')) def createManifests(mf_path): os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0')) os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0')) latest = os.path.join(mf_path, 'm3.0') os.makedirs(os.path.join(latest, 'dummy_dir3.0')) utils.symlink(latest, os.path.join(mf_path, 'active')) return (['m1.0', 'm2.0', 'm3.0'], 'm3.0') def makePackageContent(path, pkgPath, pkgPropPath): pkgFile = file(pkgPath, 'w') for index in range(10): pkgFile.write(('%s%s') % (index, index)) pkgFile.close() pkgFile = file(pkgPropPath, 'w') for index in range(10): pkgFile.write(('%s%s') % (index, index)) pkgFile.close() uname = pylons.config['agent_user_account'] TestCleanupController.rchown(path, uname) def createTestThread(serviceName): appGlobal = config['pylons.app_globals'] testTh = WaitThread(appGlobal.threadMgr, ServiceController.serviceCat(serviceName)) testTh.start() return testTh def startTestProcess(): cmd = utils.sudoCmd(["sleep", "5"], pylons.config['app_user_account']) return Popen(cmd) path1 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'foo') path2 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'bar') path3 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'agent') deleteTestDir('foo') deleteTestDir('bar') deleteTestDir('agent') # make dirs makeTestDir(path1) makeTestDir(path2) makeTestDir(path3) all_mf, active_mf = createManifests(ServiceController.manifestPath('agent')) uname = pylons.config['agent_user_account'] TestCleanupController.rchown(ServiceController.serviceRootPath(), uname) pkgDir = PackageMgr.packagePath() pkgPath = os.path.join(pkgDir, "foo.cronus") pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop") makePackageContent(pkgDir, pkgPath, pkgPropPath) # create threads testThFoo = createTestThread('foo') testThBar = createTestThread('bar') testThAgent = createTestThread('agent') # start process process = startTestProcess() # start testing LOG.debug('************ start cleanup') response = self.app.post(url(controller='cleanup', action='post')) LOG.debug ('Delete response body = ' + response.body) body = json.loads(response.body) tm = time.time() while (tm + 10 > time.time()): response = self.app.get(body['status'], expect_errors = True) LOG.debug ('Status response body = ' + response.body) body = json.loads(response.body) print body if (body['progress'] == 100): break time.sleep(0.1) # make sure the responses are correct LOG.debug('status = ' + str(response.status_int)) assert response.status_int == 200, "HTTP response != 200" time.sleep(0.1) assert not os.path.exists(path1), 'service foo does exist or is not a directory' assert not os.path.exists(path2), 'service bar does exist or is not a directory' assert os.path.exists(path3), 'service agent does NOT exist or is not a directory' assert not testThFoo.isAlive(), 'thread Foo is still alive' assert not testThBar.isAlive(), 'thread Bar is still alive' assert not testThAgent.isAlive(), 'thread Agent is still alive' assert not os.path.exists(pkgPath), 'package foo exists' assert not os.path.exists(pkgPropPath), 'package prop foo exists' assert os.path.exists(pkgDir), 'package directory does not exist' # ensure agent cleanup is proper active_mf_path = manifestutil.manifestPath('agent', active_mf) active_link = os.path.join(ServiceController.manifestPath('agent'), 'active') all_mf.remove(active_mf) actual_active_mf_path = utils.readlink(active_link) self.assertTrue(os.path.exists(active_mf_path), 'active agent manifest got deleted but shouldn\t have') self.assertTrue(os.path.exists(active_link), 'agent active link missing') self.assertEqual(active_mf_path, actual_active_mf_path, 'agent active link pointing to some wrong manifest; link broken?') for mf in all_mf: agnt_mf_path = manifestutil.manifestPath('agent', mf) self.assertFalse(os.path.exists(agnt_mf_path), 'non active agent mf %s should have been deleted' % mf)