def test_parsing_good_url(self):
# test good url
uri = 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus?query=blah#foo'
uriDict = PackageUtil.parseUri(uri, PackageMgr.packagePath())
assert uriDict['uri'] == uri
assert uriDict['scheme'] == 'http'
assert uriDict['uripath'] == '/repo/agent-0.1.2.unix.cronus'
assert uriDict['package'] == 'agent-0.1.2.unix.cronus'
assert uriDict['packageNameVersion'] == 'agent-0.1.2.unix'
assert uriDict['packageName'] == 'agent'
assert uriDict['packageVersion'] == '0.1.2.unix'
assert uriDict['packageVersionMajor'] == '0'
assert uriDict['packageVersionMinor'] == '1'
assert uriDict['packageVersionBuild'] == '2'
assert uriDict['packagePlat'] == 'unix'
assert uriDict['propName'] == 'agent-0.1.2.unix.cronus.prop'
assert uriDict[
'propUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.prop?query=blah#foo'
assert uriDict['packagePath'] == os.path.join(
PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus')
assert uriDict['inProgressPackagePath'] == os.path.join(
PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.inprogress')
assert uriDict['propPath'] == os.path.join(
PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.prop')
assert uriDict[
'torrentUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.torrent?query=blah#foo'
assert uriDict['torrentPath'] == os.path.join(
PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.torrent')
assert uriDict['torrentName'] == 'agent-0.1.2.unix.cronus.torrent'
def test_forceDeletePackages(self):
#import pdb;pdb.set_trace();
from agent.tests.unit.test_util import createManifest
if os.name == 'posix':
serviceName = 'service4'
manifestName = 'manifestb'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
minage = pylons.config['packageMgr_package_min_age']
pylons.config['packageMgr_package_min_age'] = 0.0
#appGlobal = pylons.config['pylons.app_globals']
#appGlobal.packageMgr.start()
disk_threshold = pylons.config['health_disk_usage_percent_threshold']
pylons.config['health_disk_usage_percent_threshold'] = 1
pylons.config['health_disk_usage_gc_threshold'] = 1
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 2)
PackageMgr.forceCleanUpDownloadedPkgs()
self.assertEquals(len(os.listdir(pkgs)), 0)
pylons.config['packageMgr_package_age'] = age
pylons.config['packageMgr_package_min_age'] = minage
pylons.config['health_disk_usage_percent_threshold'] = disk_threshold
def test_post5(self):
createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"])
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgB-0.6.0.unix.cronus')))
def test_cleanupOrphanedPackages2(self):
from agent.lib.utils import rchown
from agent.tests.unit.test_util import createManifest
serviceName = 'service2'
manifestName = 'manifestA'
serviceName1 = 'service3'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"],
service = serviceName, manifest = manifestName)
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"],
service = serviceName1, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
installedPkgPath = os.path.realpath(os.path.join(manifestutil.serviceRootPath() , serviceName, 'installed-packages'))
installedPkgPath1 = os.path.realpath(os.path.join(manifestutil.serviceRootPath() , serviceName1, 'installed-packages'))
uname = configutil.getAgentUser()
uid, gid = utils.getUidGid(uname)
rchown(installedPkgPath, uid, gid)
rchown(installedPkgPath1, uid, gid)
self.assertEquals(len(os.listdir(installedPkgPath)) , 1)
self.assertEquals(len(os.listdir(installedPkgPath1)) , 1)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName1, manifest = manifestName), expect_errors = True)
pylons.config['packageMgr_install_package_age'] = 0
pylons.config['packageMgr_install_package_min_age'] = 0.0
PackageMgr.cleanupOrphanedPackages()
self.assertEquals(len(os.listdir(installedPkgPath)) , 0)
self.assertEquals(len(os.listdir(installedPkgPath1)) , 0)
def test_post5(self):
createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"])
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgB-0.6.0.unix.cronus')))
def test_init(self):
# test the init routine
# make sure that the initial path is cleared out of IN Progress nodes
tmpFile = os.path.join(PackageMgr.packagePath(), 'foo.inprogress')
open(tmpFile, 'w').close()
tmpFile2 = os.path.join(PackageMgr.packagePath(), 'foo2.inprogress')
open(tmpFile2, 'w').close()
stopAgentGlobals()
startAgentGlobals()
def test_init(self):
# test the init routine
# make sure that the initial path is cleared out of IN Progress nodes
tmpFile = os.path.join(PackageMgr.packagePath(), 'foo.inprogress')
open(tmpFile, 'w').close()
tmpFile2 = os.path.join(PackageMgr.packagePath(), 'foo2.inprogress')
open(tmpFile2, 'w').close()
stopAgentGlobals()
startAgentGlobals()
def test_delete_cronus_package(self):
package = "pkgA-1.2.0.unix"
package_url = "http://repository.qa.ebay.com/cronus/test-data/agent/pkgA-1.2.0.unix.cronus"
package_path = os.path.join(PackageMgr.packagePath(),
package + '.cronus')
# case 1 - package is not present
self.assertFalse(os.path.exists(package_path))
response = self.app.delete(url(controller='cleanup',
package=package,
action='deleteCronusPackage'),
expect_errors=True)
self.assertEqual(response.status_int, 200)
# case 2 - package is present
body = json.dumps({
'package': package_url,
'packageloc': package + '.cronus'
})
response = self.app.post(url(controller='distribution',
action='startdownload',
service="dist"),
headers={'Content-Type': 'application/json'},
params=body,
expect_errors=True)
self.assertEqual(response.status_int, 200)
checkStatus(self, "http_download", response, 100)
self.assertTrue(os.path.exists(package_path))
response = self.app.delete(url(controller='cleanup',
package=package,
action='deleteCronusPackage'),
expect_errors=True)
self.assertEqual(response.status_int, 200)
self.assertFalse(os.path.exists(package_path))
def test_parsing_bad_name(self):
# test bad name
try:
PackageUtil.parseUri('http://foo.com/repo/a?gent-0.1.1.unix.cronus', PackageMgr.packagePath())
assert False, 'bad name'
except AgentException:
pass
def test_parsing_bad_extension(self):
# test bad extension
try:
PackageUtil.parseUri('http://foo.com/repo/agent-0.1.1a.cronus2', PackageMgr.packagePath())
assert False, 'bad extension'
except AgentException:
pass
def listPackages(self):
""" list content of packages folder
"""
result = {}
pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ]
result['packages'] = pkgs
return doneResult(request, response, result = result, controller = self)
def listPackages(self):
""" list content of packages folder
"""
result = {}
pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ]
result['packages'] = pkgs
return doneResult(request, response, result = result, controller = self)
def test_noAgeGarbageCollection(self):
#from agent.controllers.service import ServiceController
from agent.tests.unit.test_util import createManifest
serviceName = 'service4'
manifestName = 'manifestb'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 4)
time.sleep(40)
self.assertEquals(len(os.listdir(pkgs)), 4)
pylons.config['packageMgr_package_age'] = age
def __test_garbageCollection(self):
from agent.tests.unit.test_util import createManifest
serviceName = 'service3'
manifestName = 'manifestA'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 10
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 2)
appGlobal = pylons.config['pylons.app_globals']
appGlobal.packageMgr.start()
time.sleep(80)
self.assertEquals(len(os.listdir(pkgs)), 0)
pylons.config['packageMgr_package_age'] = age
self.assertEquals(pylons.config['packageMgr_package_age'], age)
appGlobal.packageMgr.stop()
time.sleep(20)
def test_package_min_age_during_forceDelete(self):
#import pdb;pdb.set_trace();
from agent.tests.unit.test_util import createManifest
serviceName = 'service5'
manifestName = 'manifestb'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
minage = pylons.config['packageMgr_package_min_age']
pylons.config['packageMgr_package_min_age'] = 86400
disk_threshold = pylons.config['health_disk_usage_percent_threshold']
pylons.config['health_disk_usage_percent_threshold'] = 1
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 2)
time.sleep(40)
self.assertEquals(len(os.listdir(pkgs)), 2)
pylons.config['packageMgr_package_age'] = age
pylons.config['packageMgr_package_min_age'] = minage
pylons.config['health_disk_usage_percent_threshold'] = disk_threshold
def setUp(self):
commonSetup()
self.testPkgUri = 'http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.1.0.unix.cronus'
self.localPkgName = os.path.join(PackageMgr.packagePath(), 'perlserver-1.1.0.unix.cronus')
self.threadMgr = pylons.config['pylons.app_globals'].threadMgr
appGlobals = pylons.config['pylons.app_globals']
appGlobals.packageMgr.stop()
def test_parsing_bad_extension(self):
# test bad extension
try:
PackageUtil.parseUri('http://foo.com/repo/agent-0.1.1a.cronus2',
PackageMgr.packagePath())
assert False, 'bad extension'
except AgentException:
pass
def getPkgUploadPage(self):
""" show existing packages in packages folder """
pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ]
c.content = pkgs
menuTab = "MENU_TAB_PKGS"
c.menuTab = menuTab
return render('/derived/pkgUpload.html')
def test_post2(self):
# successful post
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_parsing_bad_name(self):
# test bad name
try:
PackageUtil.parseUri(
'http://foo.com/repo/a?gent-0.1.1.unix.cronus',
PackageMgr.packagePath())
assert False, 'bad name'
except AgentException:
pass
def setUp(self):
commonSetup()
self.testPkgUri = 'http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.1.0.unix.cronus'
self.localPkgName = os.path.join(PackageMgr.packagePath(),
'perlserver-1.1.0.unix.cronus')
self.threadMgr = pylons.config['pylons.app_globals'].threadMgr
appGlobals = pylons.config['pylons.app_globals']
appGlobals.packageMgr.stop()
def test_parsing_garbage(self):
# test the url parsing
try:
PackageUtil.parseUri(
'asdfawert892375[jrgjhnjbuy43w5897t^!R$#W_R(@$^(_#$(',
PackageMgr.packagePath())
assert False, 'parse garbage'
except AgentException:
pass
def test_inprogress_pkg_download(self):
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress')
inprogressFile = open(inprogressPath, 'w')
inprogressFile.write('somegarbage')
inprogressFile.close()
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_inprogress_pkg_download(self):
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress')
inprogressFile = open(inprogressPath, 'w')
inprogressFile.write('somegarbage')
inprogressFile.close()
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
def test_parsing_good_url(self):
# test good url
uri = 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus?query=blah#foo'
uriDict = PackageUtil.parseUri(uri, PackageMgr.packagePath())
assert uriDict['uri'] == uri
assert uriDict['scheme'] == 'http'
assert uriDict['uripath'] == '/repo/agent-0.1.2.unix.cronus'
assert uriDict['package'] == 'agent-0.1.2.unix.cronus'
assert uriDict['packageNameVersion'] == 'agent-0.1.2.unix'
assert uriDict['packageName'] == 'agent'
assert uriDict['packageVersion'] == '0.1.2.unix'
assert uriDict['packageVersionMajor'] == '0'
assert uriDict['packageVersionMinor'] == '1'
assert uriDict['packageVersionBuild'] == '2'
assert uriDict['packagePlat'] == 'unix'
assert uriDict['propName'] == 'agent-0.1.2.unix.cronus.prop'
assert uriDict['propUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.prop?query=blah#foo'
assert uriDict['packagePath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus')
assert uriDict['inProgressPackagePath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.inprogress')
assert uriDict['propPath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.prop')
def _classfiy_packages(self, packages):
""" verify that all packages that need to be downloaded exist """
non_existent_packages = []
filtered_packages = []
packagePath = PackageMgr.packagePath()
for one_package in packages:
path = os.path.join(packagePath, one_package + '.cronus')
if not os.path.exists(path):
non_existent_packages.append(one_package)
else:
filtered_packages.append(one_package)
return non_existent_packages, filtered_packages
def _classfiy_packages(self, packages):
""" verify that all packages that need to be downloaded exist """
non_existent_packages = []
filtered_packages = []
packagePath = PackageMgr.packagePath()
for one_package in packages:
path = os.path.join(packagePath, one_package + '.cronus')
if not os.path.exists(path):
non_existent_packages.append(one_package)
else:
filtered_packages.append(one_package)
return non_existent_packages, filtered_packages
def test_post3_pkg_already_installed(self):
createManifest(self)
# now lets remove the manifest path
path = os.path.join(manifestutil.manifestPath('foo', 'bar'))
shutil.rmtree(path)
# now create the manifest again
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
def deletePackage(self, package):
""" secret !!hack!! API to delete one cronus package in the packages folder. """
try:
LOG.info('secret delete called for package %s' % package)
packagesPath = PackageMgr.packagePath()
thisPkgPath = os.path.join(packagesPath, package + '.cronus')
thisPropPath = os.path.join(packagesPath, package + '.cronus.prop')
thisInprogressPath = PackageUtil.inProgressPath(thisPkgPath)
PackageUtil.cleanUpPackage(thisInprogressPath, thisPkgPath, thisPropPath)
return doneResult(request, response, controller=self)
except Exception as excp:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when deleting package %s, %s - %s' %
(package, str(excp), traceback.format_exc(2)),
controller = self)
def test_post3_already_installed_manifest(self):
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']})
response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertEquals(201, response.status_int, 'Manifest Post assert')
body = json.loads(response.body)
assert response.status_int == 201, 'Manifest Post assert'
def deletePackage(self, package):
""" secret !!hack!! API to delete one cronus package in the packages folder. """
try:
LOG.info('secret delete called for package %s' % package)
packagesPath = PackageMgr.packagePath()
thisPkgPath = os.path.join(packagesPath, package + '.cronus')
thisPropPath = os.path.join(packagesPath, package + '.cronus.prop')
thisInprogressPath = PackageUtil.inProgressPath(thisPkgPath)
PackageUtil.cleanUpPackage(thisInprogressPath, thisPkgPath, thisPropPath)
return doneResult(request, response, controller=self)
except Exception as excp:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when deleting package %s, %s - %s' %
(package, str(excp), traceback.format_exc(2)),
controller = self)
def test_post3_already_installed_manifest(self):
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']})
response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertEquals(201, response.status_int, 'Manifest Post assert')
body = json.loads(response.body)
assert response.status_int == 201, 'Manifest Post assert'
def forceCleanUpDownloadedPkgs():
''' tries to cleanup packages from oldest date to latest ignoring packages downloaded in the last hour'''
import operator
import glob
import time
from agent.lib.packagemgr import PackageMgr
LOG.info('Force garbage collecting downloaded packages')
packageMount = pylons.config['agent_root']
lower_age_bound = float(pylons.config['packageMgr_package_min_age'])
cronus_pkgs = glob.glob(os.path.join(PackageMgr.packagePath(), '*.cronus'))
file_list = []
for filename in cronus_pkgs:
try:
date_file_tuple = os.path.getatime(filename), os.path.getsize(filename), filename
file_list.append(date_file_tuple)
except OSError, osErr:
LOG.error('Unable to get access time %s - %s' % (filename, osErr))
def test_package_reuse(self):
createManifest(self)
body = json.dumps({'package' : ['/packages/perlserver']})
body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
for _ in range(10):
if islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')):
break
time.sleep(1)
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')))
def test_package_reuse(self):
createManifest(self)
body = json.dumps({'package' : ['/packages/perlserver']})
body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
for _ in range(10):
if islink(os.path.join(manifestutil.manifestPath('foo', 'baz'), 'perlserver')):
break
time.sleep(1)
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'baz'), 'perlserver')))
def test_post3_pkg_already_installed(self):
createManifest(self)
# now lets remove the manifest path
path = os.path.join(ManifestController.manifestPath('foo', 'bar'))
if os.name == 'nt':
cmd = 'rm -r %s' % path
LOG.debug("running command %s" % cmd)
os.system(cmd)
else:
shutil.rmtree(path)
# now create the manifest again
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_forceDeletePackages(self):
#import pdb;pdb.set_trace();
from agent.tests.unit.test_util import createManifest
if os.name == 'posix':
serviceName = 'service4'
manifestName = 'manifestb'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
minage = pylons.config['packageMgr_package_min_age']
pylons.config['packageMgr_package_min_age'] = 0.0
#appGlobal = pylons.config['pylons.app_globals']
#appGlobal.packageMgr.start()
disk_threshold = pylons.config[
'health_disk_usage_percent_threshold']
pylons.config['health_disk_usage_percent_threshold'] = 1
pylons.config['health_disk_usage_gc_threshold'] = 1
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 4)
PackageUtil.forceCleanUpDownloadedPkgs()
self.assertEquals(len(os.listdir(pkgs)), 0)
pylons.config['packageMgr_package_age'] = age
pylons.config['packageMgr_package_min_age'] = minage
pylons.config[
'health_disk_usage_percent_threshold'] = disk_threshold
def test_same_pkg_download_parallel(self):
packages = ['http://www.stackscaling.com/agentrepo/pkgA-1.2.0.unix.cronus']
manifest1 = 'bar'
manifest2 = 'blah'
service1 = 'foo'
service2 = 'lah'
try:
for pkg in packages:
mockDownloadPkg(pkg)
path = ServiceController.manifestPath(service1)
os.makedirs(path)
path = ServiceController.installedPkgPath(service1)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service1)
os.makedirs(path)
path = ServiceController.manifestPath(service2)
os.makedirs(path)
path = ServiceController.installedPkgPath(service2)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service2)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response1.status_int == 200, 'Manifest Post assert'
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response2.status_int == 200, 'Manifest Post assert'
checkStatus(self, 'create manifest bar', response1, timeout = 25)
checkStatus(self, 'create manifest baz', response2, timeout = 25)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'pkgA')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('lah', 'blah'), 'pkgA')))
def test_noAgeGarbageCollection(self):
#from agent.controllers.service import ServiceController
from agent.tests.unit.test_util import createManifest
serviceName = 'service4'
manifestName = 'manifestb'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 2)
time.sleep(40)
self.assertEquals(len(os.listdir(pkgs)), 2)
pylons.config['packageMgr_package_age'] = age
def test_getPackagePaths(self):
serviceName = 'service1'
manifestName = 'manifestA'
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
packagePaths = ManifestController.getPackagePaths(serviceName, manifestName);
print ">>>>>>>>>>>>" + str(packagePaths)
self.assertTrue(len(packagePaths) >= 2)
bitmap = 0
for path in packagePaths:
if path.find("pkgA") >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 1
elif path.find("pkgB") >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 2
self.assertEquals(3, bitmap)
def test_getPackagePaths(self):
serviceName = 'service1'
manifestName = 'manifestA'
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
packagePaths = manifestutil.packagePaths(serviceName, manifestName);
print ">>>>>>>>>>>>" + str(packagePaths)
self.assertTrue(len(packagePaths) >= 2)
bitmap = 0
for path in packagePaths:
if path.find("pkgA") >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 1
elif path.find("pkgB") >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 2
self.assertEquals(3, bitmap)
def test_delete_cronus_package(self):
package = "pkgA-1.2.0.unix"
package_url = "http://repository.qa.ebay.com/cronus/test-data/agent/pkgA-1.2.0.unix.cronus"
package_path = os.path.join(PackageMgr.packagePath(), package + '.cronus')
# case 1 - package is not present
self.assertFalse(os.path.exists(package_path))
response = self.app.delete(url(controller='cleanup', package=package, action='deleteCronusPackage'), expect_errors = True)
self.assertEqual(response.status_int, 200)
# case 2 - package is present
body = json.dumps({'package': package_url, 'packageloc' : package + '.cronus'})
response = self.app.post(url(controller = 'distribution', action = 'startdownload', service = "dist"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertEqual(response.status_int, 200)
checkStatus(self, "http_download", response, 100)
self.assertTrue(os.path.exists(package_path))
response = self.app.delete(url(controller='cleanup', package=package, action='deleteCronusPackage'), expect_errors = True)
self.assertEqual(response.status_int, 200)
self.assertFalse(os.path.exists(package_path))
def forceCleanUpDownloadedPkgs():
''' tries to cleanup packages from oldest date to latest ignoring packages downloaded in the last hour'''
import operator
import glob
import time
from agent.lib.packagemgr import PackageMgr
LOG.info('Force garbage collecting downloaded packages')
packageMount = pylons.config['agent_root']
lower_age_bound = float(pylons.config['packageMgr_package_min_age'])
cronus_pkgs = glob.glob(
os.path.join(PackageMgr.packagePath(), '*.cronus'))
file_list = []
for filename in cronus_pkgs:
try:
date_file_tuple = os.path.getatime(filename), os.path.getsize(
filename), filename
file_list.append(date_file_tuple)
except OSError, osErr:
LOG.error('Unable to get access time %s - %s' %
(filename, osErr))
def test_same_pkg_download_parallel(self):
packages = ['http://repository.qa.ebay.com/cronus/test-data/agent/perlserver-1.0.0.unix.cronus']
manifest1 = 'bar'
manifest2 = 'blah'
service1 = 'foo'
service2 = 'lah'
try:
path = ServiceController.manifestPath(service1)
os.makedirs(path)
path = ServiceController.installedPkgPath(service1)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service1)
os.makedirs(path)
path = ServiceController.manifestPath(service2)
os.makedirs(path)
path = ServiceController.installedPkgPath(service2)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service2)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response1.status_int == 200, 'Manifest Post assert'
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response2.status_int == 200, 'Manifest Post assert'
checkStatus(self, 'create manifest bar', response1, timeout = 25)
checkStatus(self, 'create manifest baz', response2, timeout = 25)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('lah', 'blah'), 'perlserver')))
def test_cleanupPackages(self):
stopAgentGlobals()
# ok now let's create the directory structure of a bad service and packages
packagePath = PackageMgr.packagePath()
inProgPackage2 = os.path.join(packagePath, 'bar.inprogress')
inProgPackage1 = os.path.join(packagePath, 'foo.inprogress')
goodPackage1 = os.path.join(packagePath, 'baz')
goodPackage2 = os.path.join(packagePath, 'baz')
open(inProgPackage1, 'w').close()
open(inProgPackage2, 'w').close()
open(goodPackage1, 'w').close()
open(goodPackage2, 'w').close()
startAgentGlobals()
# we remove logic of inprogress file cleanup from packagemgr
# assert not os.path.exists(inProgPackage1)
# assert not os.path.exists(inProgPackage2)
assert os.path.exists(goodPackage1)
assert os.path.exists(goodPackage2)
def test_cleanupPackages(self):
stopAgentGlobals()
# ok now let's create the directory structure of a bad service and packages
packagePath = PackageMgr.packagePath()
inProgPackage2 = os.path.join(packagePath, 'bar.inprogress')
inProgPackage1 = os.path.join(packagePath, 'foo.inprogress')
goodPackage1 = os.path.join(packagePath, 'baz')
goodPackage2 = os.path.join(packagePath, 'baz')
open(inProgPackage1, 'w').close()
open(inProgPackage2, 'w').close()
open(goodPackage1, 'w').close()
open(goodPackage2, 'w').close()
startAgentGlobals()
# we remove logic of inprogress file cleanup from packagemgr
# assert not os.path.exists(inProgPackage1)
# assert not os.path.exists(inProgPackage2)
assert os.path.exists(goodPackage1)
assert os.path.exists(goodPackage2)
def test_package_min_age_during_forceDelete(self):
#import pdb;pdb.set_trace();
from agent.tests.unit.test_util import createManifest
serviceName = 'service5'
manifestName = 'manifestb'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
minage = pylons.config['packageMgr_package_min_age']
pylons.config['packageMgr_package_min_age'] = 86400
disk_threshold = pylons.config['health_disk_usage_percent_threshold']
pylons.config['health_disk_usage_percent_threshold'] = 1
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 4)
time.sleep(40)
self.assertEquals(len(os.listdir(pkgs)), 4)
pylons.config['packageMgr_package_age'] = age
pylons.config['packageMgr_package_min_age'] = minage
pylons.config['health_disk_usage_percent_threshold'] = disk_threshold
def startAgentGlobals(startThreadMgr=True,
startPackageMgr=True,
startAgentMonitor=True):
"""
Create and start the global objects
"""
# cleanup incomplete manifests
from agent.controllers.service import ServiceController
ServiceController.cleanupServices()
# stop the existing agents
stopAgentGlobals()
appGlobal = pylons.config['pylons.app_globals']
# load config override from agent .metadata.json
configutil.loadPylonConfig(pylons.config)
configutil.loadConfigOverrides()
LOG.info("Complete loading config overrides")
# build in memory security token cache
appGlobal.authztoken = str(uuid.uuid4())
appGlobal.encryptedtokens = {}
agentauth.buildTokenCache(appGlobal.authztoken)
LOG.info("Complete building security token cache")
# agent health
appGlobal.agentHealth = 'True'
appGlobal.agentHealthFactor = None
from datetime import datetime
appGlobal.agentInfo = {
'version_wiri': agenthealth.loadVersion(),
'startup_time': str(datetime.now())
}
appGlobal.osInfo = agenthealth.getOsInfo()
appGlobal.diskOk = True
appGlobal.autoRestartEnabled = asbool(
pylons.config['auto_restart_enabled'])
LOG.info("Agent health initialized")
# start thread mgr
appGlobal.threadMgr = ThreadMgr(
garbageFreq=pylons.config['threadMgr_garbage_freq'],
maxThreadAge=pylons.config['threadMgr_thread_age'])
if startThreadMgr:
appGlobal.threadMgr.start()
LOG.info("Agent thread manager started")
# start package mgr
appGlobal.packageMgr = PackageMgr(
garbageFreq=pylons.config['packageMgr_garbage_freq'],
maxPackageAge=pylons.config['packageMgr_package_age'])
if startPackageMgr:
appGlobal.packageMgr.start()
LOG.info("Agent package manager started")
# now load saved .metadata for previously saved hwPath, serverAddress, and serverAuth
enrollMeta = manifestutil.readJsonServiceMeta('agent', ['hwPath'])
hwPath = enrollMeta.get('hwPath')
if hwPath is not None:
appGlobal.hwPath = hwPath
LOG.info("Agent hwPath loaded")
#start monitor manager
from agent.lib.monitors.monitor import AgentMonitor
appGlobal.agentMonitor = AgentMonitor()
if startAgentMonitor:
appGlobal.agentMonitor.start()
LOG.info("Agent monitor started")
# start graphite udp server
# from agent.lib.monitors.graphited import GraphiteDServer
# from multiprocessing import Pipe
# graphiteEnabled = configutil.getConfigAsBool('graphite_enabled')
# if startAgentMonitor and graphiteEnabled:
# output_p, input_p = Pipe()
# appGlobal.graphited = GraphiteDServer(input_p)
# appGlobal.graphited.start()
# udpMsgProcessThread = threading.Thread(target=GraphiteDServer.read_queue, args=(output_p,))
# udpMsgProcessThread.daemon = True
# udpMsgProcessThread.start()
# LOG.info("Agent graphited started")
# Declare dictionary for storing dynamic controllers
appGlobal.dynacontrollers = dict()
# metrix manager initialization
from agent.lib.monitors.metrix_manager import MetrixManager
from agent.lib.monitors.system_monitor import SystemMonitor
appGlobal.metrixManager = MetrixManager()
appGlobal.systemMonitor = SystemMonitor()
appGlobal.metrixManager.register('OS', appGlobal.systemMonitor.getOSinfo,
1)
appGlobal.metrixManager.register('Node Name',
appGlobal.systemMonitor.getNodeName, 1)
appGlobal.metrixManager.register('Free Memory(KB)',
appGlobal.systemMonitor.getFreeMemory, 2)
appGlobal.metrixManager.register('CPU Usage(%)',
appGlobal.systemMonitor.getCpuUsage, 2)
appGlobal.metrixManager.register('Version', agenthealth.loadVersion, 5)
appGlobal.metrixManager.register('Configuration',
validate_internals.getConfigFile, 6)
appGlobal.metrixManager.register('Configuration Overrides',
configutil.getConfigOverrides, 7)
appGlobal.metrixManager.register(
'hwPath', lambda: appGlobal.hwPath
if hasattr(appGlobal, 'hwPath') else None, 5)
appGlobal.metrixManager.register(AGENT_HEALTH_VI_KEY,
lambda: appGlobal.agentHealth, 1)
appGlobal.metrixManager.register(AGENT_HEALTH_FACTOR_VI_KEY,
lambda: appGlobal.agentHealthFactor, 1)
appGlobal.metrixManager.register(OS_INFO_KEY, lambda: appGlobal.osInfo, 2)
appGlobal.metrixManager.register(AGENT_INFO_KEY,
lambda: appGlobal.agentInfo, 2)
LOG.info("Agent health metrics registered")
# start all agent modules
modulestartthread = threading.Thread(
target=ModuleController.loadModuleOnAgentStartup)
modulestartthread.start()
LOG.info("Local modules started")
# start all services with active manifest, and load dynamic controllers
servicestartthread = threading.Thread(
target=ServiceController.startServicesOnAgentStartup)
servicestartthread.start()
LOG.info("Local services started")
appGlobal.sdutil = shutdownAgent
def testDelete(self):
def deleteTestDir(service):
LOG.debug('************ service = %s' % service)
response = self.app.delete(url(controller='service',
service=service,
action='delete'),
expect_errors=True)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
# assert response.status_int == 500, "HTTP response != 500"
def makeTestDir(path):
os.makedirs(path)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'installed-packages'))
def createManifests(mf_path):
os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
latest = os.path.join(mf_path, 'm3.0')
os.makedirs(os.path.join(latest, 'dummy_dir3.0'))
utils.symlink(latest, os.path.join(mf_path, 'active'))
return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')
def makePackageContent(path, pkgPath, pkgPropPath):
pkgFile = file(pkgPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
pkgFile = file(pkgPropPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(path, uname)
def createTestThread(serviceName):
appGlobal = config['pylons.app_globals']
testTh = WaitThread(appGlobal.threadMgr,
ServiceController.serviceCat(serviceName))
testTh.start()
return testTh
def startTestProcess():
cmd = utils.sudoCmd(["sleep", "5"],
pylons.config['app_user_account'])
return Popen(cmd)
if os.name == 'nt':
LOG.warning(
'Services cleanup not supported on windows. Skipping test...')
return
path1 = os.path.join(pylons.config['agent_root'], 'service_nodes',
'foo')
path2 = os.path.join(pylons.config['agent_root'], 'service_nodes',
'bar')
path3 = os.path.join(pylons.config['agent_root'], 'service_nodes',
'agent')
deleteTestDir('foo')
deleteTestDir('bar')
deleteTestDir('agent')
# make dirs
makeTestDir(path1)
makeTestDir(path2)
makeTestDir(path3)
all_mf, active_mf = createManifests(
ServiceController.manifestPath('agent'))
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(ServiceController.serviceRootPath(),
uname)
pkgDir = PackageMgr.packagePath()
pkgPath = os.path.join(pkgDir, "foo.cronus")
pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop")
makePackageContent(pkgDir, pkgPath, pkgPropPath)
# create threads
testThFoo = createTestThread('foo')
testThBar = createTestThread('bar')
testThAgent = createTestThread('agent')
# start process
process = startTestProcess()
# start testing
LOG.debug('************ start cleanup')
response = self.app.post(url(controller='cleanup', action='post'))
LOG.debug('Delete response body = ' + response.body)
body = json.loads(response.body)
tm = time.time()
while (tm + 10 > time.time()):
response = self.app.get(body['status'], expect_errors=True)
LOG.debug('Status response body = ' + response.body)
body = json.loads(response.body)
print body
if (body['progress'] == 100):
break
time.sleep(0.1)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
time.sleep(0.1)
assert not os.path.exists(
path1), 'service foo does exist or is not a directory'
assert not os.path.exists(
path2), 'service bar does exist or is not a directory'
assert os.path.exists(
path3), 'service agent does NOT exist or is not a directory'
assert not testThFoo.isAlive(), 'thread Foo is still alive'
assert not testThBar.isAlive(), 'thread Bar is still alive'
assert not testThAgent.isAlive(), 'thread Agent is still alive'
assert not os.path.exists(pkgPath), 'package foo exists'
assert not os.path.exists(pkgPropPath), 'package prop foo exists'
assert os.path.exists(pkgDir), 'package directory does not exist'
# ensure agent cleanup is proper
active_mf_path = ManifestController.manifestPath('agent', active_mf)
active_link = os.path.join(ServiceController.manifestPath('agent'),
'active')
all_mf.remove(active_mf)
actual_active_mf_path = utils.readlink(active_link)
self.assertTrue(
os.path.exists(active_mf_path),
'active agent manifest got deleted but shouldn\t have')
self.assertTrue(os.path.exists(active_link),
'agent active link missing')
self.assertEqual(
active_mf_path, actual_active_mf_path,
'agent active link pointing to some wrong manifest; link broken?')
for mf in all_mf:
agnt_mf_path = ManifestController.manifestPath('agent', mf)
self.assertFalse(
os.path.exists(agnt_mf_path),
'non active agent mf %s should have been deleted' % mf)
def parseUri(packageUri, path=None, packageloc=None):
""" static method to parse the uri
@throws - AgentException
"""
if (path == None):
from agent.lib.packagemgr import PackageMgr
path = PackageMgr.packagePath()
uri = urlparse.urlparse(packageUri)
if (uri.scheme != 'http'):
raise AgentException(
Errors.PACKAGE_SCHEME_ERROR,
'uri (%s) scheme(%s) not supported' % (packageUri, uri.scheme))
if (uri.path == ''):
raise AgentException(Errors.PACKAGE_PATH_ERROR,
'uri (%s) path is empty' % (packageUri))
# now parse the path. get the name and then verify that it matches the convention
if packageloc is not None:
# check if torrent download
if uri.path.endswith('.torrent'):
if not packageloc.endswith('.torrent'):
packageloc = packageloc + '.torrent'
packName = packageloc
else:
packName = uri.path.rpartition('/')[2]
match = PackageUtil.nameRe.match(packName)
if (match == None or match.group(0) != packName):
raise AgentException(
Errors.PACKAGE_PATH_ERROR,
'cannot find package name in path %s' % (uri.path))
# ok now we can fill out the dictionary
uriDict = {}
uriDict['uri'] = packageUri
uriDict['scheme'] = uri.scheme
uriDict['uripath'] = uri.path
uriDict['package'] = match.group(1)
uriDict['packageNameVersion'] = match.group(2)
uriDict['inProgressPackage'] = PackageUtil.inProgressPath(
uriDict['package'])
uriDict['packageName'] = match.group(3)
uriDict['propName'] = uriDict['package'] + '.prop'
uriDict['packageVersion'] = match.group(4)
uriDict['packageVersionMajor'] = match.group(5)
uriDict['packageVersionMinor'] = match.group(6)
uriDict['packageVersionBuild'] = match.group(7)
uriDict['packagePlat'] = match.group(8)
# path specific attributes
uriDict['packagePath'] = os.path.join(path, uriDict['package'])
uriDict['inProgressPackagePath'] = os.path.join(
path, uriDict['inProgressPackage'])
uriDict['propPath'] = os.path.join(path, uriDict['propName'])
# check if bittorrent transfer and define the necessary keys
if match.group(9) is not None and match.group(9) == '.torrent':
uriDict['scheme'] = 'bittorrent'
uriDict['torrentName'] = uriDict['package'] + '.torrent'
uriDict['torrentPath'] = os.path.join(path, uriDict['torrentName'])
# calculate prop url
# append path with .prop - mons: leaving the below code in place in case we support other protocols in future
uriScheme = uri.scheme
if (uriScheme !=
"http"): #only use http to download .prop and .torrent files
uriScheme = "http"
if uri.path.endswith('.torrent'):
uripath = uri.path.strip('.torrent')
else:
uripath = uri.path
propParseResult = urlparse.ParseResult(uriScheme, uri.netloc,
uripath + '.prop', uri.params,
uri.query, uri.fragment)
propUri = urlparse.urlunparse(propParseResult)
uriDict['propUri'] = propUri
# calculate the torrent url
# append path with .torrent
torrentParseResult = urlparse.ParseResult(uriScheme, uri.netloc,
uripath + '.torrent',
uri.params, uri.query,
uri.fragment)
torrentUri = urlparse.urlunparse(torrentParseResult)
uriDict['torrentUri'] = torrentUri
return uriDict
def test_parsing_garbage(self):
# test the url parsing
try:
PackageUtil.parseUri('asdfawert892375[jrgjhnjbuy43w5897t^!R$#W_R(@$^(_#$(', PackageMgr.packagePath())
assert False, 'parse garbage'
except AgentException:
pass
def __deletePackages(self):
""" delete all packages from pakcage directory """
package_path = PackageMgr.packagePath()
ServiceDelete.deleteFolderContents(package_path, onlyChildren = True)
def runAgentGc(self):
""" force agent GC """
LOG.info('Received runAgentGc request')
PackageMgr.runAgentGc()
return doneResult(request, response, controller = self)
def parseUri(packageUri, path = None, packageloc = None):
""" static method to parse the uri
@throws - AgentException
"""
if (path == None):
from agent.lib.packagemgr import PackageMgr
path = PackageMgr.packagePath()
uri = urlparse.urlparse(packageUri)
if (uri.scheme != 'http'):
raise AgentException(Errors.PACKAGE_SCHEME_ERROR,
'uri (%s) scheme(%s) not supported' % (packageUri, uri.scheme))
if (uri.path == ''):
raise AgentException(Errors.PACKAGE_PATH_ERROR,
'uri (%s) path is empty' % (packageUri))
# now parse the path. get the name and then verify that it matches the convention
if packageloc is not None:
# check if torrent download
if uri.path.endswith('.torrent'):
if not packageloc.endswith('.torrent'):
packageloc = packageloc + '.torrent'
packName = packageloc
else:
packName = uri.path.rpartition('/')[2]
match = PackageUtil.nameRe.match(packName)
if (match == None or match.group(0) != packName):
raise AgentException(Errors.PACKAGE_PATH_ERROR,
'cannot find package name in path %s' % (uri.path))
# ok now we can fill out the dictionary
uriDict = {}
uriDict['uri'] = packageUri
uriDict['scheme'] = uri.scheme
uriDict['uripath'] = uri.path
uriDict['package'] = match.group(1)
uriDict['packageNameVersion'] = match.group(2)
uriDict['inProgressPackage'] = PackageUtil.inProgressPath(uriDict['package'])
uriDict['packageName'] = match.group(3)
uriDict['propName'] = uriDict['package'] + '.prop'
uriDict['packageVersion'] = match.group(4)
uriDict['packageVersionMajor'] = match.group(5)
uriDict['packageVersionMinor'] = match.group(6)
uriDict['packageVersionBuild'] = match.group(7)
uriDict['packagePlat'] = match.group(8)
# path specific attributes
uriDict['packagePath'] = os.path.join(path, uriDict['package'])
uriDict['inProgressPackagePath'] = os.path.join(path, uriDict['inProgressPackage'])
uriDict['propPath'] = os.path.join(path, uriDict['propName'])
# check if bittorrent transfer and define the necessary keys
if match.group(9) is not None and match.group(9) == '.torrent':
uriDict['scheme'] = 'bittorrent'
uriDict['torrentName'] = uriDict['package'] + '.torrent'
uriDict['torrentPath'] = os.path.join(path, uriDict['torrentName'])
# calculate prop url
# append path with .prop - mons: leaving the below code in place in case we support other protocols in future
uriScheme = uri.scheme
if (uriScheme != "http"): #only use http to download .prop and .torrent files
uriScheme = "http"
if uri.path.endswith('.torrent'):
uripath = uri.path.strip('.torrent')
else:
uripath = uri.path
propParseResult = urlparse.ParseResult(uriScheme, uri.netloc, uripath + '.prop',
uri.params, uri.query, uri.fragment)
propUri = urlparse.urlunparse(propParseResult)
uriDict['propUri'] = propUri
# calculate the torrent url
# append path with .torrent
torrentParseResult = urlparse.ParseResult(uriScheme, uri.netloc, uripath + '.torrent',
uri.params, uri.query, uri.fragment)
torrentUri = urlparse.urlunparse(torrentParseResult)
uriDict['torrentUri'] = torrentUri
return uriDict
def __deletePackages(self):
""" delete all packages from pakcage directory """
package_path = PackageMgr.packagePath()
ServiceDelete.deleteFolderContents(package_path, onlyChildren=True)
