def test_parsing_good_url(self): # test good url uri = 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus?query=blah#foo' uriDict = PackageUtil.parseUri(uri, PackageMgr.packagePath()) assert uriDict['uri'] == uri assert uriDict['scheme'] == 'http' assert uriDict['uripath'] == '/repo/agent-0.1.2.unix.cronus' assert uriDict['package'] == 'agent-0.1.2.unix.cronus' assert uriDict['packageNameVersion'] == 'agent-0.1.2.unix' assert uriDict['packageName'] == 'agent' assert uriDict['packageVersion'] == '0.1.2.unix' assert uriDict['packageVersionMajor'] == '0' assert uriDict['packageVersionMinor'] == '1' assert uriDict['packageVersionBuild'] == '2' assert uriDict['packagePlat'] == 'unix' assert uriDict['propName'] == 'agent-0.1.2.unix.cronus.prop' assert uriDict[ 'propUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.prop?query=blah#foo' assert uriDict['packagePath'] == os.path.join( PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus') assert uriDict['inProgressPackagePath'] == os.path.join( PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.inprogress') assert uriDict['propPath'] == os.path.join( PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.prop') assert uriDict[ 'torrentUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.torrent?query=blah#foo' assert uriDict['torrentPath'] == os.path.join( PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.torrent') assert uriDict['torrentName'] == 'agent-0.1.2.unix.cronus.torrent'
def test_forceDeletePackages(self): #import pdb;pdb.set_trace(); from agent.tests.unit.test_util import createManifest if os.name == 'posix': serviceName = 'service4' manifestName = 'manifestb' try: createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"], service = serviceName, manifest = manifestName) except Exception as ex: print 'exception thrown during mf %s' % str(ex) self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True) age = pylons.config['packageMgr_package_age'] pylons.config['packageMgr_package_age'] = 86400 minage = pylons.config['packageMgr_package_min_age'] pylons.config['packageMgr_package_min_age'] = 0.0 #appGlobal = pylons.config['pylons.app_globals'] #appGlobal.packageMgr.start() disk_threshold = pylons.config['health_disk_usage_percent_threshold'] pylons.config['health_disk_usage_percent_threshold'] = 1 pylons.config['health_disk_usage_gc_threshold'] = 1 pkgs = PackageMgr.packagePath() self.assertEquals(len(os.listdir(pkgs)), 2) PackageMgr.forceCleanUpDownloadedPkgs() self.assertEquals(len(os.listdir(pkgs)), 0) pylons.config['packageMgr_package_age'] = age pylons.config['packageMgr_package_min_age'] = minage pylons.config['health_disk_usage_percent_threshold'] = disk_threshold
def test_post5(self): createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"]) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgB-0.6.0.unix.cronus')))
def test_cleanupOrphanedPackages2(self): from agent.lib.utils import rchown from agent.tests.unit.test_util import createManifest serviceName = 'service2' manifestName = 'manifestA' serviceName1 = 'service3' try: createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"], service = serviceName, manifest = manifestName) createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"], service = serviceName1, manifest = manifestName) except Exception as ex: print 'exception thrown during mf %s' % str(ex) installedPkgPath = os.path.realpath(os.path.join(manifestutil.serviceRootPath() , serviceName, 'installed-packages')) installedPkgPath1 = os.path.realpath(os.path.join(manifestutil.serviceRootPath() , serviceName1, 'installed-packages')) uname = configutil.getAgentUser() uid, gid = utils.getUidGid(uname) rchown(installedPkgPath, uid, gid) rchown(installedPkgPath1, uid, gid) self.assertEquals(len(os.listdir(installedPkgPath)) , 1) self.assertEquals(len(os.listdir(installedPkgPath1)) , 1) self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True) self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName1, manifest = manifestName), expect_errors = True) pylons.config['packageMgr_install_package_age'] = 0 pylons.config['packageMgr_install_package_min_age'] = 0.0 PackageMgr.cleanupOrphanedPackages() self.assertEquals(len(os.listdir(installedPkgPath)) , 0) self.assertEquals(len(os.listdir(installedPkgPath1)) , 0)
def test_post5(self): createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"]) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgB-0.6.0.unix.cronus')))
def test_init(self): # test the init routine # make sure that the initial path is cleared out of IN Progress nodes tmpFile = os.path.join(PackageMgr.packagePath(), 'foo.inprogress') open(tmpFile, 'w').close() tmpFile2 = os.path.join(PackageMgr.packagePath(), 'foo2.inprogress') open(tmpFile2, 'w').close() stopAgentGlobals() startAgentGlobals()
def test_init(self): # test the init routine # make sure that the initial path is cleared out of IN Progress nodes tmpFile = os.path.join(PackageMgr.packagePath(), 'foo.inprogress') open(tmpFile, 'w').close() tmpFile2 = os.path.join(PackageMgr.packagePath(), 'foo2.inprogress') open(tmpFile2, 'w').close() stopAgentGlobals() startAgentGlobals()
def test_delete_cronus_package(self): package = "pkgA-1.2.0.unix" package_url = "http://repository.qa.ebay.com/cronus/test-data/agent/pkgA-1.2.0.unix.cronus" package_path = os.path.join(PackageMgr.packagePath(), package + '.cronus') # case 1 - package is not present self.assertFalse(os.path.exists(package_path)) response = self.app.delete(url(controller='cleanup', package=package, action='deleteCronusPackage'), expect_errors=True) self.assertEqual(response.status_int, 200) # case 2 - package is present body = json.dumps({ 'package': package_url, 'packageloc': package + '.cronus' }) response = self.app.post(url(controller='distribution', action='startdownload', service="dist"), headers={'Content-Type': 'application/json'}, params=body, expect_errors=True) self.assertEqual(response.status_int, 200) checkStatus(self, "http_download", response, 100) self.assertTrue(os.path.exists(package_path)) response = self.app.delete(url(controller='cleanup', package=package, action='deleteCronusPackage'), expect_errors=True) self.assertEqual(response.status_int, 200) self.assertFalse(os.path.exists(package_path))
def test_parsing_bad_name(self): # test bad name try: PackageUtil.parseUri('http://foo.com/repo/a?gent-0.1.1.unix.cronus', PackageMgr.packagePath()) assert False, 'bad name' except AgentException: pass
def test_parsing_bad_extension(self): # test bad extension try: PackageUtil.parseUri('http://foo.com/repo/agent-0.1.1a.cronus2', PackageMgr.packagePath()) assert False, 'bad extension' except AgentException: pass
def listPackages(self): """ list content of packages folder """ result = {} pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ] result['packages'] = pkgs return doneResult(request, response, result = result, controller = self)
def listPackages(self): """ list content of packages folder """ result = {} pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ] result['packages'] = pkgs return doneResult(request, response, result = result, controller = self)
def test_noAgeGarbageCollection(self): #from agent.controllers.service import ServiceController from agent.tests.unit.test_util import createManifest serviceName = 'service4' manifestName = 'manifestb' try: createManifest( self, packages=[ "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus" ], service=serviceName, manifest=manifestName) except Exception as ex: print 'exception thrown during mf %s' % str(ex) self.app.delete(url(controller='manifest', action='delete', service=serviceName, manifest=manifestName), expect_errors=True) age = pylons.config['packageMgr_package_age'] pylons.config['packageMgr_package_age'] = 86400 pkgs = PackageMgr.packagePath() self.assertEquals(len(os.listdir(pkgs)), 4) time.sleep(40) self.assertEquals(len(os.listdir(pkgs)), 4) pylons.config['packageMgr_package_age'] = age
def __test_garbageCollection(self): from agent.tests.unit.test_util import createManifest serviceName = 'service3' manifestName = 'manifestA' try: createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"], service = serviceName, manifest = manifestName) except Exception as ex: print 'exception thrown during mf %s' % str(ex) self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True) age = pylons.config['packageMgr_package_age'] pylons.config['packageMgr_package_age'] = 10 pkgs = PackageMgr.packagePath() self.assertEquals(len(os.listdir(pkgs)), 2) appGlobal = pylons.config['pylons.app_globals'] appGlobal.packageMgr.start() time.sleep(80) self.assertEquals(len(os.listdir(pkgs)), 0) pylons.config['packageMgr_package_age'] = age self.assertEquals(pylons.config['packageMgr_package_age'], age) appGlobal.packageMgr.stop() time.sleep(20)
def test_package_min_age_during_forceDelete(self): #import pdb;pdb.set_trace(); from agent.tests.unit.test_util import createManifest serviceName = 'service5' manifestName = 'manifestb' try: createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"], service = serviceName, manifest = manifestName) except Exception as ex: print 'exception thrown during mf %s' % str(ex) self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True) age = pylons.config['packageMgr_package_age'] pylons.config['packageMgr_package_age'] = 86400 minage = pylons.config['packageMgr_package_min_age'] pylons.config['packageMgr_package_min_age'] = 86400 disk_threshold = pylons.config['health_disk_usage_percent_threshold'] pylons.config['health_disk_usage_percent_threshold'] = 1 pkgs = PackageMgr.packagePath() self.assertEquals(len(os.listdir(pkgs)), 2) time.sleep(40) self.assertEquals(len(os.listdir(pkgs)), 2) pylons.config['packageMgr_package_age'] = age pylons.config['packageMgr_package_min_age'] = minage pylons.config['health_disk_usage_percent_threshold'] = disk_threshold
def setUp(self): commonSetup() self.testPkgUri = 'http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.1.0.unix.cronus' self.localPkgName = os.path.join(PackageMgr.packagePath(), 'perlserver-1.1.0.unix.cronus') self.threadMgr = pylons.config['pylons.app_globals'].threadMgr appGlobals = pylons.config['pylons.app_globals'] appGlobals.packageMgr.stop()
def test_parsing_bad_extension(self): # test bad extension try: PackageUtil.parseUri('http://foo.com/repo/agent-0.1.1a.cronus2', PackageMgr.packagePath()) assert False, 'bad extension' except AgentException: pass
def getPkgUploadPage(self): """ show existing packages in packages folder """ pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ] c.content = pkgs menuTab = "MENU_TAB_PKGS" c.menuTab = menuTab return render('/derived/pkgUpload.html')
def test_post2(self): # successful post createManifest(self) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix', 'cronus', 'scripts', 'activate'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_parsing_bad_name(self): # test bad name try: PackageUtil.parseUri( 'http://foo.com/repo/a?gent-0.1.1.unix.cronus', PackageMgr.packagePath()) assert False, 'bad name' except AgentException: pass
def setUp(self): commonSetup() self.testPkgUri = 'http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.1.0.unix.cronus' self.localPkgName = os.path.join(PackageMgr.packagePath(), 'perlserver-1.1.0.unix.cronus') self.threadMgr = pylons.config['pylons.app_globals'].threadMgr appGlobals = pylons.config['pylons.app_globals'] appGlobals.packageMgr.stop()
def test_parsing_garbage(self): # test the url parsing try: PackageUtil.parseUri( 'asdfawert892375[jrgjhnjbuy43w5897t^!R$#W_R(@$^(_#$(', PackageMgr.packagePath()) assert False, 'parse garbage' except AgentException: pass
def test_inprogress_pkg_download(self): service = 'foo' try: path = ServiceController.manifestPath(service) os.makedirs(path) path = ServiceController.installedPkgPath(service) os.makedirs(path) path = ServiceController.downloadedPkgPath(service) os.makedirs(path) inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress') inprogressFile = open(inprogressPath, 'w') inprogressFile.write('somegarbage') inprogressFile.close() except Exception as excep: LOG.warning('got an OS Exception - %s' % str(excep)) createManifest(self) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_inprogress_pkg_download(self): service = 'foo' try: path = ServiceController.manifestPath(service) os.makedirs(path) path = ServiceController.installedPkgPath(service) os.makedirs(path) path = ServiceController.downloadedPkgPath(service) os.makedirs(path) inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress') inprogressFile = open(inprogressPath, 'w') inprogressFile.write('somegarbage') inprogressFile.close() except Exception as excep: LOG.warning('got an OS Exception - %s' % str(excep)) createManifest(self) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
def test_parsing_good_url(self): # test good url uri = 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus?query=blah#foo' uriDict = PackageUtil.parseUri(uri, PackageMgr.packagePath()) assert uriDict['uri'] == uri assert uriDict['scheme'] == 'http' assert uriDict['uripath'] == '/repo/agent-0.1.2.unix.cronus' assert uriDict['package'] == 'agent-0.1.2.unix.cronus' assert uriDict['packageNameVersion'] == 'agent-0.1.2.unix' assert uriDict['packageName'] == 'agent' assert uriDict['packageVersion'] == '0.1.2.unix' assert uriDict['packageVersionMajor'] == '0' assert uriDict['packageVersionMinor'] == '1' assert uriDict['packageVersionBuild'] == '2' assert uriDict['packagePlat'] == 'unix' assert uriDict['propName'] == 'agent-0.1.2.unix.cronus.prop' assert uriDict['propUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.prop?query=blah#foo' assert uriDict['packagePath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus') assert uriDict['inProgressPackagePath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.inprogress') assert uriDict['propPath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.prop')
def _classfiy_packages(self, packages): """ verify that all packages that need to be downloaded exist """ non_existent_packages = [] filtered_packages = [] packagePath = PackageMgr.packagePath() for one_package in packages: path = os.path.join(packagePath, one_package + '.cronus') if not os.path.exists(path): non_existent_packages.append(one_package) else: filtered_packages.append(one_package) return non_existent_packages, filtered_packages
def _classfiy_packages(self, packages): """ verify that all packages that need to be downloaded exist """ non_existent_packages = [] filtered_packages = [] packagePath = PackageMgr.packagePath() for one_package in packages: path = os.path.join(packagePath, one_package + '.cronus') if not os.path.exists(path): non_existent_packages.append(one_package) else: filtered_packages.append(one_package) return non_existent_packages, filtered_packages
def test_post3_pkg_already_installed(self): createManifest(self) # now lets remove the manifest path path = os.path.join(manifestutil.manifestPath('foo', 'bar')) shutil.rmtree(path) # now create the manifest again createManifest(self) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix', 'cronus', 'scripts', 'activate'))) self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
def deletePackage(self, package): """ secret !!hack!! API to delete one cronus package in the packages folder. """ try: LOG.info('secret delete called for package %s' % package) packagesPath = PackageMgr.packagePath() thisPkgPath = os.path.join(packagesPath, package + '.cronus') thisPropPath = os.path.join(packagesPath, package + '.cronus.prop') thisInprogressPath = PackageUtil.inProgressPath(thisPkgPath) PackageUtil.cleanUpPackage(thisInprogressPath, thisPkgPath, thisPropPath) return doneResult(request, response, controller=self) except Exception as excp: return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg = 'Unknown error when deleting package %s, %s - %s' % (package, str(excp), traceback.format_exc(2)), controller = self)
def test_post3_already_installed_manifest(self): createManifest(self) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix', 'cronus', 'scripts', 'activate'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver'))) body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']}) response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"), headers = {'Content-Type' : 'application/json'}, params = body, expect_errors = True) self.assertEquals(201, response.status_int, 'Manifest Post assert') body = json.loads(response.body) assert response.status_int == 201, 'Manifest Post assert'
def deletePackage(self, package): """ secret !!hack!! API to delete one cronus package in the packages folder. """ try: LOG.info('secret delete called for package %s' % package) packagesPath = PackageMgr.packagePath() thisPkgPath = os.path.join(packagesPath, package + '.cronus') thisPropPath = os.path.join(packagesPath, package + '.cronus.prop') thisInprogressPath = PackageUtil.inProgressPath(thisPkgPath) PackageUtil.cleanUpPackage(thisInprogressPath, thisPkgPath, thisPropPath) return doneResult(request, response, controller=self) except Exception as excp: return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg = 'Unknown error when deleting package %s, %s - %s' % (package, str(excp), traceback.format_exc(2)), controller = self)
def test_post3_already_installed_manifest(self): createManifest(self) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix', 'cronus', 'scripts', 'activate'))) self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver'))) body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']}) response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"), headers = {'Content-Type' : 'application/json'}, params = body, expect_errors = True) self.assertEquals(201, response.status_int, 'Manifest Post assert') body = json.loads(response.body) assert response.status_int == 201, 'Manifest Post assert'
def forceCleanUpDownloadedPkgs(): ''' tries to cleanup packages from oldest date to latest ignoring packages downloaded in the last hour''' import operator import glob import time from agent.lib.packagemgr import PackageMgr LOG.info('Force garbage collecting downloaded packages') packageMount = pylons.config['agent_root'] lower_age_bound = float(pylons.config['packageMgr_package_min_age']) cronus_pkgs = glob.glob(os.path.join(PackageMgr.packagePath(), '*.cronus')) file_list = [] for filename in cronus_pkgs: try: date_file_tuple = os.path.getatime(filename), os.path.getsize(filename), filename file_list.append(date_file_tuple) except OSError, osErr: LOG.error('Unable to get access time %s - %s' % (filename, osErr))
def test_package_reuse(self): createManifest(self) body = json.dumps({'package' : ['/packages/perlserver']}) body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"), headers = {'Content-Type' : 'application/json'}, params = body, expect_errors = True) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix', 'cronus', 'scripts', 'activate'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver'))) for _ in range(10): if islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')): break time.sleep(1) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')))
def test_package_reuse(self): createManifest(self) body = json.dumps({'package' : ['/packages/perlserver']}) body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"), headers = {'Content-Type' : 'application/json'}, params = body, expect_errors = True) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix', 'cronus', 'scripts', 'activate'))) self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver'))) for _ in range(10): if islink(os.path.join(manifestutil.manifestPath('foo', 'baz'), 'perlserver')): break time.sleep(1) self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'baz'), 'perlserver')))
def test_post3_pkg_already_installed(self): createManifest(self) # now lets remove the manifest path path = os.path.join(ManifestController.manifestPath('foo', 'bar')) if os.name == 'nt': cmd = 'rm -r %s' % path LOG.debug("running command %s" % cmd) os.system(cmd) else: shutil.rmtree(path) # now create the manifest again createManifest(self) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix', 'cronus', 'scripts', 'activate'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_forceDeletePackages(self): #import pdb;pdb.set_trace(); from agent.tests.unit.test_util import createManifest if os.name == 'posix': serviceName = 'service4' manifestName = 'manifestb' try: createManifest( self, packages=[ "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus" ], service=serviceName, manifest=manifestName) except Exception as ex: print 'exception thrown during mf %s' % str(ex) self.app.delete(url(controller='manifest', action='delete', service=serviceName, manifest=manifestName), expect_errors=True) age = pylons.config['packageMgr_package_age'] pylons.config['packageMgr_package_age'] = 86400 minage = pylons.config['packageMgr_package_min_age'] pylons.config['packageMgr_package_min_age'] = 0.0 #appGlobal = pylons.config['pylons.app_globals'] #appGlobal.packageMgr.start() disk_threshold = pylons.config[ 'health_disk_usage_percent_threshold'] pylons.config['health_disk_usage_percent_threshold'] = 1 pylons.config['health_disk_usage_gc_threshold'] = 1 pkgs = PackageMgr.packagePath() self.assertEquals(len(os.listdir(pkgs)), 4) PackageUtil.forceCleanUpDownloadedPkgs() self.assertEquals(len(os.listdir(pkgs)), 0) pylons.config['packageMgr_package_age'] = age pylons.config['packageMgr_package_min_age'] = minage pylons.config[ 'health_disk_usage_percent_threshold'] = disk_threshold
def test_same_pkg_download_parallel(self): packages = ['http://www.stackscaling.com/agentrepo/pkgA-1.2.0.unix.cronus'] manifest1 = 'bar' manifest2 = 'blah' service1 = 'foo' service2 = 'lah' try: for pkg in packages: mockDownloadPkg(pkg) path = ServiceController.manifestPath(service1) os.makedirs(path) path = ServiceController.installedPkgPath(service1) os.makedirs(path) path = ServiceController.downloadedPkgPath(service1) os.makedirs(path) path = ServiceController.manifestPath(service2) os.makedirs(path) path = ServiceController.installedPkgPath(service2) os.makedirs(path) path = ServiceController.downloadedPkgPath(service2) os.makedirs(path) except Exception as excep: LOG.warning('got an OS Exception - %s' % str(excep)) body = json.dumps({'package' : packages}) response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1), headers = {'Content-Type' : 'application/json'}, params = body) assert response1.status_int == 200, 'Manifest Post assert' response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2), headers = {'Content-Type' : 'application/json'}, params = body) assert response2.status_int == 200, 'Manifest Post assert' checkStatus(self, 'create manifest bar', response1, timeout = 25) checkStatus(self, 'create manifest baz', response2, timeout = 25) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus'))) self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'pkgA'))) self.assertTrue(islink(os.path.join(manifestutil.manifestPath('lah', 'blah'), 'pkgA')))
def test_noAgeGarbageCollection(self): #from agent.controllers.service import ServiceController from agent.tests.unit.test_util import createManifest serviceName = 'service4' manifestName = 'manifestb' try: createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"], service = serviceName, manifest = manifestName) except Exception as ex: print 'exception thrown during mf %s' % str(ex) self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True) age = pylons.config['packageMgr_package_age'] pylons.config['packageMgr_package_age'] = 86400 pkgs = PackageMgr.packagePath() self.assertEquals(len(os.listdir(pkgs)), 2) time.sleep(40) self.assertEquals(len(os.listdir(pkgs)), 2) pylons.config['packageMgr_package_age'] = age
def test_getPackagePaths(self): serviceName = 'service1' manifestName = 'manifestA' createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"], service = serviceName, manifest = manifestName) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus'))) packagePaths = ManifestController.getPackagePaths(serviceName, manifestName); print ">>>>>>>>>>>>" + str(packagePaths) self.assertTrue(len(packagePaths) >= 2) bitmap = 0 for path in packagePaths: if path.find("pkgA") >= 0: self.assertTrue(os.path.isdir(path)) bitmap |= 1 elif path.find("pkgB") >= 0: self.assertTrue(os.path.isdir(path)) bitmap |= 2 self.assertEquals(3, bitmap)
def test_getPackagePaths(self): serviceName = 'service1' manifestName = 'manifestA' createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"], service = serviceName, manifest = manifestName) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus'))) packagePaths = manifestutil.packagePaths(serviceName, manifestName); print ">>>>>>>>>>>>" + str(packagePaths) self.assertTrue(len(packagePaths) >= 2) bitmap = 0 for path in packagePaths: if path.find("pkgA") >= 0: self.assertTrue(os.path.isdir(path)) bitmap |= 1 elif path.find("pkgB") >= 0: self.assertTrue(os.path.isdir(path)) bitmap |= 2 self.assertEquals(3, bitmap)
def test_delete_cronus_package(self): package = "pkgA-1.2.0.unix" package_url = "http://repository.qa.ebay.com/cronus/test-data/agent/pkgA-1.2.0.unix.cronus" package_path = os.path.join(PackageMgr.packagePath(), package + '.cronus') # case 1 - package is not present self.assertFalse(os.path.exists(package_path)) response = self.app.delete(url(controller='cleanup', package=package, action='deleteCronusPackage'), expect_errors = True) self.assertEqual(response.status_int, 200) # case 2 - package is present body = json.dumps({'package': package_url, 'packageloc' : package + '.cronus'}) response = self.app.post(url(controller = 'distribution', action = 'startdownload', service = "dist"), headers = {'Content-Type' : 'application/json'}, params = body, expect_errors = True) self.assertEqual(response.status_int, 200) checkStatus(self, "http_download", response, 100) self.assertTrue(os.path.exists(package_path)) response = self.app.delete(url(controller='cleanup', package=package, action='deleteCronusPackage'), expect_errors = True) self.assertEqual(response.status_int, 200) self.assertFalse(os.path.exists(package_path))
def forceCleanUpDownloadedPkgs(): ''' tries to cleanup packages from oldest date to latest ignoring packages downloaded in the last hour''' import operator import glob import time from agent.lib.packagemgr import PackageMgr LOG.info('Force garbage collecting downloaded packages') packageMount = pylons.config['agent_root'] lower_age_bound = float(pylons.config['packageMgr_package_min_age']) cronus_pkgs = glob.glob( os.path.join(PackageMgr.packagePath(), '*.cronus')) file_list = [] for filename in cronus_pkgs: try: date_file_tuple = os.path.getatime(filename), os.path.getsize( filename), filename file_list.append(date_file_tuple) except OSError, osErr: LOG.error('Unable to get access time %s - %s' % (filename, osErr))
def test_same_pkg_download_parallel(self): packages = ['http://repository.qa.ebay.com/cronus/test-data/agent/perlserver-1.0.0.unix.cronus'] manifest1 = 'bar' manifest2 = 'blah' service1 = 'foo' service2 = 'lah' try: path = ServiceController.manifestPath(service1) os.makedirs(path) path = ServiceController.installedPkgPath(service1) os.makedirs(path) path = ServiceController.downloadedPkgPath(service1) os.makedirs(path) path = ServiceController.manifestPath(service2) os.makedirs(path) path = ServiceController.installedPkgPath(service2) os.makedirs(path) path = ServiceController.downloadedPkgPath(service2) os.makedirs(path) except Exception as excep: LOG.warning('got an OS Exception - %s' % str(excep)) body = json.dumps({'package' : packages}) response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1), headers = {'Content-Type' : 'application/json'}, params = body) assert response1.status_int == 200, 'Manifest Post assert' response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2), headers = {'Content-Type' : 'application/json'}, params = body) assert response2.status_int == 200, 'Manifest Post assert' checkStatus(self, 'create manifest bar', response1, timeout = 25) checkStatus(self, 'create manifest baz', response2, timeout = 25) self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver'))) self.assertTrue(islink(os.path.join(ManifestController.manifestPath('lah', 'blah'), 'perlserver')))
def test_cleanupPackages(self): stopAgentGlobals() # ok now let's create the directory structure of a bad service and packages packagePath = PackageMgr.packagePath() inProgPackage2 = os.path.join(packagePath, 'bar.inprogress') inProgPackage1 = os.path.join(packagePath, 'foo.inprogress') goodPackage1 = os.path.join(packagePath, 'baz') goodPackage2 = os.path.join(packagePath, 'baz') open(inProgPackage1, 'w').close() open(inProgPackage2, 'w').close() open(goodPackage1, 'w').close() open(goodPackage2, 'w').close() startAgentGlobals() # we remove logic of inprogress file cleanup from packagemgr # assert not os.path.exists(inProgPackage1) # assert not os.path.exists(inProgPackage2) assert os.path.exists(goodPackage1) assert os.path.exists(goodPackage2)
def test_cleanupPackages(self): stopAgentGlobals() # ok now let's create the directory structure of a bad service and packages packagePath = PackageMgr.packagePath() inProgPackage2 = os.path.join(packagePath, 'bar.inprogress') inProgPackage1 = os.path.join(packagePath, 'foo.inprogress') goodPackage1 = os.path.join(packagePath, 'baz') goodPackage2 = os.path.join(packagePath, 'baz') open(inProgPackage1, 'w').close() open(inProgPackage2, 'w').close() open(goodPackage1, 'w').close() open(goodPackage2, 'w').close() startAgentGlobals() # we remove logic of inprogress file cleanup from packagemgr # assert not os.path.exists(inProgPackage1) # assert not os.path.exists(inProgPackage2) assert os.path.exists(goodPackage1) assert os.path.exists(goodPackage2)
def test_package_min_age_during_forceDelete(self): #import pdb;pdb.set_trace(); from agent.tests.unit.test_util import createManifest serviceName = 'service5' manifestName = 'manifestb' try: createManifest( self, packages=[ "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus", "http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus" ], service=serviceName, manifest=manifestName) except Exception as ex: print 'exception thrown during mf %s' % str(ex) self.app.delete(url(controller='manifest', action='delete', service=serviceName, manifest=manifestName), expect_errors=True) age = pylons.config['packageMgr_package_age'] pylons.config['packageMgr_package_age'] = 86400 minage = pylons.config['packageMgr_package_min_age'] pylons.config['packageMgr_package_min_age'] = 86400 disk_threshold = pylons.config['health_disk_usage_percent_threshold'] pylons.config['health_disk_usage_percent_threshold'] = 1 pkgs = PackageMgr.packagePath() self.assertEquals(len(os.listdir(pkgs)), 4) time.sleep(40) self.assertEquals(len(os.listdir(pkgs)), 4) pylons.config['packageMgr_package_age'] = age pylons.config['packageMgr_package_min_age'] = minage pylons.config['health_disk_usage_percent_threshold'] = disk_threshold
def startAgentGlobals(startThreadMgr=True, startPackageMgr=True, startAgentMonitor=True): """ Create and start the global objects """ # cleanup incomplete manifests from agent.controllers.service import ServiceController ServiceController.cleanupServices() # stop the existing agents stopAgentGlobals() appGlobal = pylons.config['pylons.app_globals'] # load config override from agent .metadata.json configutil.loadPylonConfig(pylons.config) configutil.loadConfigOverrides() LOG.info("Complete loading config overrides") # build in memory security token cache appGlobal.authztoken = str(uuid.uuid4()) appGlobal.encryptedtokens = {} agentauth.buildTokenCache(appGlobal.authztoken) LOG.info("Complete building security token cache") # agent health appGlobal.agentHealth = 'True' appGlobal.agentHealthFactor = None from datetime import datetime appGlobal.agentInfo = { 'version_wiri': agenthealth.loadVersion(), 'startup_time': str(datetime.now()) } appGlobal.osInfo = agenthealth.getOsInfo() appGlobal.diskOk = True appGlobal.autoRestartEnabled = asbool( pylons.config['auto_restart_enabled']) LOG.info("Agent health initialized") # start thread mgr appGlobal.threadMgr = ThreadMgr( garbageFreq=pylons.config['threadMgr_garbage_freq'], maxThreadAge=pylons.config['threadMgr_thread_age']) if startThreadMgr: appGlobal.threadMgr.start() LOG.info("Agent thread manager started") # start package mgr appGlobal.packageMgr = PackageMgr( garbageFreq=pylons.config['packageMgr_garbage_freq'], maxPackageAge=pylons.config['packageMgr_package_age']) if startPackageMgr: appGlobal.packageMgr.start() LOG.info("Agent package manager started") # now load saved .metadata for previously saved hwPath, serverAddress, and serverAuth enrollMeta = manifestutil.readJsonServiceMeta('agent', ['hwPath']) hwPath = enrollMeta.get('hwPath') if hwPath is not None: appGlobal.hwPath = hwPath LOG.info("Agent hwPath loaded") #start monitor manager from agent.lib.monitors.monitor import AgentMonitor appGlobal.agentMonitor = AgentMonitor() if startAgentMonitor: appGlobal.agentMonitor.start() LOG.info("Agent monitor started") # start graphite udp server # from agent.lib.monitors.graphited import GraphiteDServer # from multiprocessing import Pipe # graphiteEnabled = configutil.getConfigAsBool('graphite_enabled') # if startAgentMonitor and graphiteEnabled: # output_p, input_p = Pipe() # appGlobal.graphited = GraphiteDServer(input_p) # appGlobal.graphited.start() # udpMsgProcessThread = threading.Thread(target=GraphiteDServer.read_queue, args=(output_p,)) # udpMsgProcessThread.daemon = True # udpMsgProcessThread.start() # LOG.info("Agent graphited started") # Declare dictionary for storing dynamic controllers appGlobal.dynacontrollers = dict() # metrix manager initialization from agent.lib.monitors.metrix_manager import MetrixManager from agent.lib.monitors.system_monitor import SystemMonitor appGlobal.metrixManager = MetrixManager() appGlobal.systemMonitor = SystemMonitor() appGlobal.metrixManager.register('OS', appGlobal.systemMonitor.getOSinfo, 1) appGlobal.metrixManager.register('Node Name', appGlobal.systemMonitor.getNodeName, 1) appGlobal.metrixManager.register('Free Memory(KB)', appGlobal.systemMonitor.getFreeMemory, 2) appGlobal.metrixManager.register('CPU Usage(%)', appGlobal.systemMonitor.getCpuUsage, 2) appGlobal.metrixManager.register('Version', agenthealth.loadVersion, 5) appGlobal.metrixManager.register('Configuration', validate_internals.getConfigFile, 6) appGlobal.metrixManager.register('Configuration Overrides', configutil.getConfigOverrides, 7) appGlobal.metrixManager.register( 'hwPath', lambda: appGlobal.hwPath if hasattr(appGlobal, 'hwPath') else None, 5) appGlobal.metrixManager.register(AGENT_HEALTH_VI_KEY, lambda: appGlobal.agentHealth, 1) appGlobal.metrixManager.register(AGENT_HEALTH_FACTOR_VI_KEY, lambda: appGlobal.agentHealthFactor, 1) appGlobal.metrixManager.register(OS_INFO_KEY, lambda: appGlobal.osInfo, 2) appGlobal.metrixManager.register(AGENT_INFO_KEY, lambda: appGlobal.agentInfo, 2) LOG.info("Agent health metrics registered") # start all agent modules modulestartthread = threading.Thread( target=ModuleController.loadModuleOnAgentStartup) modulestartthread.start() LOG.info("Local modules started") # start all services with active manifest, and load dynamic controllers servicestartthread = threading.Thread( target=ServiceController.startServicesOnAgentStartup) servicestartthread.start() LOG.info("Local services started") appGlobal.sdutil = shutdownAgent
def testDelete(self): def deleteTestDir(service): LOG.debug('************ service = %s' % service) response = self.app.delete(url(controller='service', service=service, action='delete'), expect_errors=True) # make sure the responses are correct LOG.debug('status = ' + str(response.status_int)) # assert response.status_int == 500, "HTTP response != 500" def makeTestDir(path): os.makedirs(path) os.makedirs(os.path.join(path, 'manifests')) os.makedirs(os.path.join(path, 'installed-packages')) def createManifests(mf_path): os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0')) os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0')) latest = os.path.join(mf_path, 'm3.0') os.makedirs(os.path.join(latest, 'dummy_dir3.0')) utils.symlink(latest, os.path.join(mf_path, 'active')) return (['m1.0', 'm2.0', 'm3.0'], 'm3.0') def makePackageContent(path, pkgPath, pkgPropPath): pkgFile = file(pkgPath, 'w') for index in range(10): pkgFile.write(('%s%s') % (index, index)) pkgFile.close() pkgFile = file(pkgPropPath, 'w') for index in range(10): pkgFile.write(('%s%s') % (index, index)) pkgFile.close() uname = pylons.config['agent_user_account'] TestCleanupController.rchown(path, uname) def createTestThread(serviceName): appGlobal = config['pylons.app_globals'] testTh = WaitThread(appGlobal.threadMgr, ServiceController.serviceCat(serviceName)) testTh.start() return testTh def startTestProcess(): cmd = utils.sudoCmd(["sleep", "5"], pylons.config['app_user_account']) return Popen(cmd) if os.name == 'nt': LOG.warning( 'Services cleanup not supported on windows. Skipping test...') return path1 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'foo') path2 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'bar') path3 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'agent') deleteTestDir('foo') deleteTestDir('bar') deleteTestDir('agent') # make dirs makeTestDir(path1) makeTestDir(path2) makeTestDir(path3) all_mf, active_mf = createManifests( ServiceController.manifestPath('agent')) uname = pylons.config['agent_user_account'] TestCleanupController.rchown(ServiceController.serviceRootPath(), uname) pkgDir = PackageMgr.packagePath() pkgPath = os.path.join(pkgDir, "foo.cronus") pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop") makePackageContent(pkgDir, pkgPath, pkgPropPath) # create threads testThFoo = createTestThread('foo') testThBar = createTestThread('bar') testThAgent = createTestThread('agent') # start process process = startTestProcess() # start testing LOG.debug('************ start cleanup') response = self.app.post(url(controller='cleanup', action='post')) LOG.debug('Delete response body = ' + response.body) body = json.loads(response.body) tm = time.time() while (tm + 10 > time.time()): response = self.app.get(body['status'], expect_errors=True) LOG.debug('Status response body = ' + response.body) body = json.loads(response.body) print body if (body['progress'] == 100): break time.sleep(0.1) # make sure the responses are correct LOG.debug('status = ' + str(response.status_int)) assert response.status_int == 200, "HTTP response != 200" time.sleep(0.1) assert not os.path.exists( path1), 'service foo does exist or is not a directory' assert not os.path.exists( path2), 'service bar does exist or is not a directory' assert os.path.exists( path3), 'service agent does NOT exist or is not a directory' assert not testThFoo.isAlive(), 'thread Foo is still alive' assert not testThBar.isAlive(), 'thread Bar is still alive' assert not testThAgent.isAlive(), 'thread Agent is still alive' assert not os.path.exists(pkgPath), 'package foo exists' assert not os.path.exists(pkgPropPath), 'package prop foo exists' assert os.path.exists(pkgDir), 'package directory does not exist' # ensure agent cleanup is proper active_mf_path = ManifestController.manifestPath('agent', active_mf) active_link = os.path.join(ServiceController.manifestPath('agent'), 'active') all_mf.remove(active_mf) actual_active_mf_path = utils.readlink(active_link) self.assertTrue( os.path.exists(active_mf_path), 'active agent manifest got deleted but shouldn\t have') self.assertTrue(os.path.exists(active_link), 'agent active link missing') self.assertEqual( active_mf_path, actual_active_mf_path, 'agent active link pointing to some wrong manifest; link broken?') for mf in all_mf: agnt_mf_path = ManifestController.manifestPath('agent', mf) self.assertFalse( os.path.exists(agnt_mf_path), 'non active agent mf %s should have been deleted' % mf)
def parseUri(packageUri, path=None, packageloc=None): """ static method to parse the uri @throws - AgentException """ if (path == None): from agent.lib.packagemgr import PackageMgr path = PackageMgr.packagePath() uri = urlparse.urlparse(packageUri) if (uri.scheme != 'http'): raise AgentException( Errors.PACKAGE_SCHEME_ERROR, 'uri (%s) scheme(%s) not supported' % (packageUri, uri.scheme)) if (uri.path == ''): raise AgentException(Errors.PACKAGE_PATH_ERROR, 'uri (%s) path is empty' % (packageUri)) # now parse the path. get the name and then verify that it matches the convention if packageloc is not None: # check if torrent download if uri.path.endswith('.torrent'): if not packageloc.endswith('.torrent'): packageloc = packageloc + '.torrent' packName = packageloc else: packName = uri.path.rpartition('/')[2] match = PackageUtil.nameRe.match(packName) if (match == None or match.group(0) != packName): raise AgentException( Errors.PACKAGE_PATH_ERROR, 'cannot find package name in path %s' % (uri.path)) # ok now we can fill out the dictionary uriDict = {} uriDict['uri'] = packageUri uriDict['scheme'] = uri.scheme uriDict['uripath'] = uri.path uriDict['package'] = match.group(1) uriDict['packageNameVersion'] = match.group(2) uriDict['inProgressPackage'] = PackageUtil.inProgressPath( uriDict['package']) uriDict['packageName'] = match.group(3) uriDict['propName'] = uriDict['package'] + '.prop' uriDict['packageVersion'] = match.group(4) uriDict['packageVersionMajor'] = match.group(5) uriDict['packageVersionMinor'] = match.group(6) uriDict['packageVersionBuild'] = match.group(7) uriDict['packagePlat'] = match.group(8) # path specific attributes uriDict['packagePath'] = os.path.join(path, uriDict['package']) uriDict['inProgressPackagePath'] = os.path.join( path, uriDict['inProgressPackage']) uriDict['propPath'] = os.path.join(path, uriDict['propName']) # check if bittorrent transfer and define the necessary keys if match.group(9) is not None and match.group(9) == '.torrent': uriDict['scheme'] = 'bittorrent' uriDict['torrentName'] = uriDict['package'] + '.torrent' uriDict['torrentPath'] = os.path.join(path, uriDict['torrentName']) # calculate prop url # append path with .prop - mons: leaving the below code in place in case we support other protocols in future uriScheme = uri.scheme if (uriScheme != "http"): #only use http to download .prop and .torrent files uriScheme = "http" if uri.path.endswith('.torrent'): uripath = uri.path.strip('.torrent') else: uripath = uri.path propParseResult = urlparse.ParseResult(uriScheme, uri.netloc, uripath + '.prop', uri.params, uri.query, uri.fragment) propUri = urlparse.urlunparse(propParseResult) uriDict['propUri'] = propUri # calculate the torrent url # append path with .torrent torrentParseResult = urlparse.ParseResult(uriScheme, uri.netloc, uripath + '.torrent', uri.params, uri.query, uri.fragment) torrentUri = urlparse.urlunparse(torrentParseResult) uriDict['torrentUri'] = torrentUri return uriDict
def test_parsing_garbage(self): # test the url parsing try: PackageUtil.parseUri('asdfawert892375[jrgjhnjbuy43w5897t^!R$#W_R(@$^(_#$(', PackageMgr.packagePath()) assert False, 'parse garbage' except AgentException: pass
def __deletePackages(self): """ delete all packages from pakcage directory """ package_path = PackageMgr.packagePath() ServiceDelete.deleteFolderContents(package_path, onlyChildren = True)
def runAgentGc(self): """ force agent GC """ LOG.info('Received runAgentGc request') PackageMgr.runAgentGc() return doneResult(request, response, controller = self)
def parseUri(packageUri, path = None, packageloc = None): """ static method to parse the uri @throws - AgentException """ if (path == None): from agent.lib.packagemgr import PackageMgr path = PackageMgr.packagePath() uri = urlparse.urlparse(packageUri) if (uri.scheme != 'http'): raise AgentException(Errors.PACKAGE_SCHEME_ERROR, 'uri (%s) scheme(%s) not supported' % (packageUri, uri.scheme)) if (uri.path == ''): raise AgentException(Errors.PACKAGE_PATH_ERROR, 'uri (%s) path is empty' % (packageUri)) # now parse the path. get the name and then verify that it matches the convention if packageloc is not None: # check if torrent download if uri.path.endswith('.torrent'): if not packageloc.endswith('.torrent'): packageloc = packageloc + '.torrent' packName = packageloc else: packName = uri.path.rpartition('/')[2] match = PackageUtil.nameRe.match(packName) if (match == None or match.group(0) != packName): raise AgentException(Errors.PACKAGE_PATH_ERROR, 'cannot find package name in path %s' % (uri.path)) # ok now we can fill out the dictionary uriDict = {} uriDict['uri'] = packageUri uriDict['scheme'] = uri.scheme uriDict['uripath'] = uri.path uriDict['package'] = match.group(1) uriDict['packageNameVersion'] = match.group(2) uriDict['inProgressPackage'] = PackageUtil.inProgressPath(uriDict['package']) uriDict['packageName'] = match.group(3) uriDict['propName'] = uriDict['package'] + '.prop' uriDict['packageVersion'] = match.group(4) uriDict['packageVersionMajor'] = match.group(5) uriDict['packageVersionMinor'] = match.group(6) uriDict['packageVersionBuild'] = match.group(7) uriDict['packagePlat'] = match.group(8) # path specific attributes uriDict['packagePath'] = os.path.join(path, uriDict['package']) uriDict['inProgressPackagePath'] = os.path.join(path, uriDict['inProgressPackage']) uriDict['propPath'] = os.path.join(path, uriDict['propName']) # check if bittorrent transfer and define the necessary keys if match.group(9) is not None and match.group(9) == '.torrent': uriDict['scheme'] = 'bittorrent' uriDict['torrentName'] = uriDict['package'] + '.torrent' uriDict['torrentPath'] = os.path.join(path, uriDict['torrentName']) # calculate prop url # append path with .prop - mons: leaving the below code in place in case we support other protocols in future uriScheme = uri.scheme if (uriScheme != "http"): #only use http to download .prop and .torrent files uriScheme = "http" if uri.path.endswith('.torrent'): uripath = uri.path.strip('.torrent') else: uripath = uri.path propParseResult = urlparse.ParseResult(uriScheme, uri.netloc, uripath + '.prop', uri.params, uri.query, uri.fragment) propUri = urlparse.urlunparse(propParseResult) uriDict['propUri'] = propUri # calculate the torrent url # append path with .torrent torrentParseResult = urlparse.ParseResult(uriScheme, uri.netloc, uripath + '.torrent', uri.params, uri.query, uri.fragment) torrentUri = urlparse.urlunparse(torrentParseResult) uriDict['torrentUri'] = torrentUri return uriDict
def __deletePackages(self): """ delete all packages from pakcage directory """ package_path = PackageMgr.packagePath() ServiceDelete.deleteFolderContents(package_path, onlyChildren=True)