def test_post5(self):
createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"])
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgB-0.6.0.unix.cronus')))
def test_parsing_good_url(self):
# test good url
uri = 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus?query=blah#foo'
uriDict = PackageUtil.parseUri(uri, PackageMgr.packagePath())
assert uriDict['uri'] == uri
assert uriDict['scheme'] == 'http'
assert uriDict['uripath'] == '/repo/agent-0.1.2.unix.cronus'
assert uriDict['package'] == 'agent-0.1.2.unix.cronus'
assert uriDict['packageNameVersion'] == 'agent-0.1.2.unix'
assert uriDict['packageName'] == 'agent'
assert uriDict['packageVersion'] == '0.1.2.unix'
assert uriDict['packageVersionMajor'] == '0'
assert uriDict['packageVersionMinor'] == '1'
assert uriDict['packageVersionBuild'] == '2'
assert uriDict['packagePlat'] == 'unix'
assert uriDict['propName'] == 'agent-0.1.2.unix.cronus.prop'
assert uriDict[
'propUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.prop?query=blah#foo'
assert uriDict['packagePath'] == os.path.join(
PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus')
assert uriDict['inProgressPackagePath'] == os.path.join(
PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.inprogress')
assert uriDict['propPath'] == os.path.join(
PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.prop')
assert uriDict[
'torrentUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.torrent?query=blah#foo'
assert uriDict['torrentPath'] == os.path.join(
PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.torrent')
assert uriDict['torrentName'] == 'agent-0.1.2.unix.cronus.torrent'
def test_post5(self):
createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"])
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgB-0.6.0.unix.cronus')))
def test_init(self):
# test the init routine
# make sure that the initial path is cleared out of IN Progress nodes
tmpFile = os.path.join(PackageMgr.packagePath(), 'foo.inprogress')
open(tmpFile, 'w').close()
tmpFile2 = os.path.join(PackageMgr.packagePath(), 'foo2.inprogress')
open(tmpFile2, 'w').close()
stopAgentGlobals()
startAgentGlobals()
def test_init(self):
# test the init routine
# make sure that the initial path is cleared out of IN Progress nodes
tmpFile = os.path.join(PackageMgr.packagePath(), 'foo.inprogress')
open(tmpFile, 'w').close()
tmpFile2 = os.path.join(PackageMgr.packagePath(), 'foo2.inprogress')
open(tmpFile2, 'w').close()
stopAgentGlobals()
startAgentGlobals()
def listPackages(self):
""" list content of packages folder
"""
result = {}
pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ]
result['packages'] = pkgs
return doneResult(request, response, result = result, controller = self)
def listPackages(self):
""" list content of packages folder
"""
result = {}
pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ]
result['packages'] = pkgs
return doneResult(request, response, result = result, controller = self)
def test_noAgeGarbageCollection(self):
#from agent.controllers.service import ServiceController
from agent.tests.unit.test_util import createManifest
serviceName = 'service4'
manifestName = 'manifestb'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 4)
time.sleep(40)
self.assertEquals(len(os.listdir(pkgs)), 4)
pylons.config['packageMgr_package_age'] = age
def test_delete_cronus_package(self):
package = "pkgA-1.2.0.unix"
package_url = "http://repository.qa.ebay.com/cronus/test-data/agent/pkgA-1.2.0.unix.cronus"
package_path = os.path.join(PackageMgr.packagePath(),
package + '.cronus')
# case 1 - package is not present
self.assertFalse(os.path.exists(package_path))
response = self.app.delete(url(controller='cleanup',
package=package,
action='deleteCronusPackage'),
expect_errors=True)
self.assertEqual(response.status_int, 200)
# case 2 - package is present
body = json.dumps({
'package': package_url,
'packageloc': package + '.cronus'
})
response = self.app.post(url(controller='distribution',
action='startdownload',
service="dist"),
headers={'Content-Type': 'application/json'},
params=body,
expect_errors=True)
self.assertEqual(response.status_int, 200)
checkStatus(self, "http_download", response, 100)
self.assertTrue(os.path.exists(package_path))
response = self.app.delete(url(controller='cleanup',
package=package,
action='deleteCronusPackage'),
expect_errors=True)
self.assertEqual(response.status_int, 200)
self.assertFalse(os.path.exists(package_path))
def __test_garbageCollection(self):
from agent.tests.unit.test_util import createManifest
serviceName = 'service3'
manifestName = 'manifestA'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 10
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 2)
appGlobal = pylons.config['pylons.app_globals']
appGlobal.packageMgr.start()
time.sleep(80)
self.assertEquals(len(os.listdir(pkgs)), 0)
pylons.config['packageMgr_package_age'] = age
self.assertEquals(pylons.config['packageMgr_package_age'], age)
appGlobal.packageMgr.stop()
time.sleep(20)
def test_forceDeletePackages(self):
#import pdb;pdb.set_trace();
from agent.tests.unit.test_util import createManifest
if os.name == 'posix':
serviceName = 'service4'
manifestName = 'manifestb'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
minage = pylons.config['packageMgr_package_min_age']
pylons.config['packageMgr_package_min_age'] = 0.0
#appGlobal = pylons.config['pylons.app_globals']
#appGlobal.packageMgr.start()
disk_threshold = pylons.config['health_disk_usage_percent_threshold']
pylons.config['health_disk_usage_percent_threshold'] = 1
pylons.config['health_disk_usage_gc_threshold'] = 1
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 2)
PackageMgr.forceCleanUpDownloadedPkgs()
self.assertEquals(len(os.listdir(pkgs)), 0)
pylons.config['packageMgr_package_age'] = age
pylons.config['packageMgr_package_min_age'] = minage
pylons.config['health_disk_usage_percent_threshold'] = disk_threshold
def test_parsing_bad_extension(self):
# test bad extension
try:
PackageUtil.parseUri('http://foo.com/repo/agent-0.1.1a.cronus2', PackageMgr.packagePath())
assert False, 'bad extension'
except AgentException:
pass
def test_package_min_age_during_forceDelete(self):
#import pdb;pdb.set_trace();
from agent.tests.unit.test_util import createManifest
serviceName = 'service5'
manifestName = 'manifestb'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
minage = pylons.config['packageMgr_package_min_age']
pylons.config['packageMgr_package_min_age'] = 86400
disk_threshold = pylons.config['health_disk_usage_percent_threshold']
pylons.config['health_disk_usage_percent_threshold'] = 1
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 2)
time.sleep(40)
self.assertEquals(len(os.listdir(pkgs)), 2)
pylons.config['packageMgr_package_age'] = age
pylons.config['packageMgr_package_min_age'] = minage
pylons.config['health_disk_usage_percent_threshold'] = disk_threshold
def test_parsing_bad_name(self):
# test bad name
try:
PackageUtil.parseUri('http://foo.com/repo/a?gent-0.1.1.unix.cronus', PackageMgr.packagePath())
assert False, 'bad name'
except AgentException:
pass
def setUp(self):
commonSetup()
self.testPkgUri = 'http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.1.0.unix.cronus'
self.localPkgName = os.path.join(PackageMgr.packagePath(), 'perlserver-1.1.0.unix.cronus')
self.threadMgr = pylons.config['pylons.app_globals'].threadMgr
appGlobals = pylons.config['pylons.app_globals']
appGlobals.packageMgr.stop()
def test_parsing_bad_extension(self):
# test bad extension
try:
PackageUtil.parseUri('http://foo.com/repo/agent-0.1.1a.cronus2',
PackageMgr.packagePath())
assert False, 'bad extension'
except AgentException:
pass
def getPkgUploadPage(self):
""" show existing packages in packages folder """
pkgs = [ os.path.basename(packageDir) for packageDir in os.listdir(PackageMgr.packagePath()) ]
c.content = pkgs
menuTab = "MENU_TAB_PKGS"
c.menuTab = menuTab
return render('/derived/pkgUpload.html')
def test_post2(self):
# successful post
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_parsing_bad_name(self):
# test bad name
try:
PackageUtil.parseUri(
'http://foo.com/repo/a?gent-0.1.1.unix.cronus',
PackageMgr.packagePath())
assert False, 'bad name'
except AgentException:
pass
def setUp(self):
commonSetup()
self.testPkgUri = 'http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.1.0.unix.cronus'
self.localPkgName = os.path.join(PackageMgr.packagePath(),
'perlserver-1.1.0.unix.cronus')
self.threadMgr = pylons.config['pylons.app_globals'].threadMgr
appGlobals = pylons.config['pylons.app_globals']
appGlobals.packageMgr.stop()
def test_inprogress_pkg_download(self):
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress')
inprogressFile = open(inprogressPath, 'w')
inprogressFile.write('somegarbage')
inprogressFile.close()
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_parsing_garbage(self):
# test the url parsing
try:
PackageUtil.parseUri(
'asdfawert892375[jrgjhnjbuy43w5897t^!R$#W_R(@$^(_#$(',
PackageMgr.packagePath())
assert False, 'parse garbage'
except AgentException:
pass
def test_inprogress_pkg_download(self):
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress')
inprogressFile = open(inprogressPath, 'w')
inprogressFile.write('somegarbage')
inprogressFile.close()
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
def test_parsing_good_url(self):
# test good url
uri = 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus?query=blah#foo'
uriDict = PackageUtil.parseUri(uri, PackageMgr.packagePath())
assert uriDict['uri'] == uri
assert uriDict['scheme'] == 'http'
assert uriDict['uripath'] == '/repo/agent-0.1.2.unix.cronus'
assert uriDict['package'] == 'agent-0.1.2.unix.cronus'
assert uriDict['packageNameVersion'] == 'agent-0.1.2.unix'
assert uriDict['packageName'] == 'agent'
assert uriDict['packageVersion'] == '0.1.2.unix'
assert uriDict['packageVersionMajor'] == '0'
assert uriDict['packageVersionMinor'] == '1'
assert uriDict['packageVersionBuild'] == '2'
assert uriDict['packagePlat'] == 'unix'
assert uriDict['propName'] == 'agent-0.1.2.unix.cronus.prop'
assert uriDict['propUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.prop?query=blah#foo'
assert uriDict['packagePath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus')
assert uriDict['inProgressPackagePath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.inprogress')
assert uriDict['propPath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.prop')
def _classfiy_packages(self, packages):
""" verify that all packages that need to be downloaded exist """
non_existent_packages = []
filtered_packages = []
packagePath = PackageMgr.packagePath()
for one_package in packages:
path = os.path.join(packagePath, one_package + '.cronus')
if not os.path.exists(path):
non_existent_packages.append(one_package)
else:
filtered_packages.append(one_package)
return non_existent_packages, filtered_packages
def _classfiy_packages(self, packages):
""" verify that all packages that need to be downloaded exist """
non_existent_packages = []
filtered_packages = []
packagePath = PackageMgr.packagePath()
for one_package in packages:
path = os.path.join(packagePath, one_package + '.cronus')
if not os.path.exists(path):
non_existent_packages.append(one_package)
else:
filtered_packages.append(one_package)
return non_existent_packages, filtered_packages
def test_post3_pkg_already_installed(self):
createManifest(self)
# now lets remove the manifest path
path = os.path.join(manifestutil.manifestPath('foo', 'bar'))
shutil.rmtree(path)
# now create the manifest again
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
def deletePackage(self, package):
""" secret !!hack!! API to delete one cronus package in the packages folder. """
try:
LOG.info('secret delete called for package %s' % package)
packagesPath = PackageMgr.packagePath()
thisPkgPath = os.path.join(packagesPath, package + '.cronus')
thisPropPath = os.path.join(packagesPath, package + '.cronus.prop')
thisInprogressPath = PackageUtil.inProgressPath(thisPkgPath)
PackageUtil.cleanUpPackage(thisInprogressPath, thisPkgPath, thisPropPath)
return doneResult(request, response, controller=self)
except Exception as excp:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when deleting package %s, %s - %s' %
(package, str(excp), traceback.format_exc(2)),
controller = self)
def test_post3_already_installed_manifest(self):
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']})
response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertEquals(201, response.status_int, 'Manifest Post assert')
body = json.loads(response.body)
assert response.status_int == 201, 'Manifest Post assert'
def deletePackage(self, package):
""" secret !!hack!! API to delete one cronus package in the packages folder. """
try:
LOG.info('secret delete called for package %s' % package)
packagesPath = PackageMgr.packagePath()
thisPkgPath = os.path.join(packagesPath, package + '.cronus')
thisPropPath = os.path.join(packagesPath, package + '.cronus.prop')
thisInprogressPath = PackageUtil.inProgressPath(thisPkgPath)
PackageUtil.cleanUpPackage(thisInprogressPath, thisPkgPath, thisPropPath)
return doneResult(request, response, controller=self)
except Exception as excp:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when deleting package %s, %s - %s' %
(package, str(excp), traceback.format_exc(2)),
controller = self)
def test_post3_already_installed_manifest(self):
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']})
response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertEquals(201, response.status_int, 'Manifest Post assert')
body = json.loads(response.body)
assert response.status_int == 201, 'Manifest Post assert'
def test_package_reuse(self):
createManifest(self)
body = json.dumps({'package' : ['/packages/perlserver']})
body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
for _ in range(10):
if islink(os.path.join(manifestutil.manifestPath('foo', 'baz'), 'perlserver')):
break
time.sleep(1)
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'baz'), 'perlserver')))
def test_package_reuse(self):
createManifest(self)
body = json.dumps({'package' : ['/packages/perlserver']})
body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
for _ in range(10):
if islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')):
break
time.sleep(1)
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')))
def forceCleanUpDownloadedPkgs():
''' tries to cleanup packages from oldest date to latest ignoring packages downloaded in the last hour'''
import operator
import glob
import time
from agent.lib.packagemgr import PackageMgr
LOG.info('Force garbage collecting downloaded packages')
packageMount = pylons.config['agent_root']
lower_age_bound = float(pylons.config['packageMgr_package_min_age'])
cronus_pkgs = glob.glob(os.path.join(PackageMgr.packagePath(), '*.cronus'))
file_list = []
for filename in cronus_pkgs:
try:
date_file_tuple = os.path.getatime(filename), os.path.getsize(filename), filename
file_list.append(date_file_tuple)
except OSError, osErr:
LOG.error('Unable to get access time %s - %s' % (filename, osErr))
def test_post3_pkg_already_installed(self):
createManifest(self)
# now lets remove the manifest path
path = os.path.join(ManifestController.manifestPath('foo', 'bar'))
if os.name == 'nt':
cmd = 'rm -r %s' % path
LOG.debug("running command %s" % cmd)
os.system(cmd)
else:
shutil.rmtree(path)
# now create the manifest again
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_forceDeletePackages(self):
#import pdb;pdb.set_trace();
from agent.tests.unit.test_util import createManifest
if os.name == 'posix':
serviceName = 'service4'
manifestName = 'manifestb'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
minage = pylons.config['packageMgr_package_min_age']
pylons.config['packageMgr_package_min_age'] = 0.0
#appGlobal = pylons.config['pylons.app_globals']
#appGlobal.packageMgr.start()
disk_threshold = pylons.config[
'health_disk_usage_percent_threshold']
pylons.config['health_disk_usage_percent_threshold'] = 1
pylons.config['health_disk_usage_gc_threshold'] = 1
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 4)
PackageUtil.forceCleanUpDownloadedPkgs()
self.assertEquals(len(os.listdir(pkgs)), 0)
pylons.config['packageMgr_package_age'] = age
pylons.config['packageMgr_package_min_age'] = minage
pylons.config[
'health_disk_usage_percent_threshold'] = disk_threshold
def test_same_pkg_download_parallel(self):
packages = ['http://www.stackscaling.com/agentrepo/pkgA-1.2.0.unix.cronus']
manifest1 = 'bar'
manifest2 = 'blah'
service1 = 'foo'
service2 = 'lah'
try:
for pkg in packages:
mockDownloadPkg(pkg)
path = ServiceController.manifestPath(service1)
os.makedirs(path)
path = ServiceController.installedPkgPath(service1)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service1)
os.makedirs(path)
path = ServiceController.manifestPath(service2)
os.makedirs(path)
path = ServiceController.installedPkgPath(service2)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service2)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response1.status_int == 200, 'Manifest Post assert'
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response2.status_int == 200, 'Manifest Post assert'
checkStatus(self, 'create manifest bar', response1, timeout = 25)
checkStatus(self, 'create manifest baz', response2, timeout = 25)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'pkgA')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('lah', 'blah'), 'pkgA')))
def test_noAgeGarbageCollection(self):
#from agent.controllers.service import ServiceController
from agent.tests.unit.test_util import createManifest
serviceName = 'service4'
manifestName = 'manifestb'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 2)
time.sleep(40)
self.assertEquals(len(os.listdir(pkgs)), 2)
pylons.config['packageMgr_package_age'] = age
def test_getPackagePaths(self):
serviceName = 'service1'
manifestName = 'manifestA'
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
packagePaths = manifestutil.packagePaths(serviceName, manifestName);
print ">>>>>>>>>>>>" + str(packagePaths)
self.assertTrue(len(packagePaths) >= 2)
bitmap = 0
for path in packagePaths:
if path.find("pkgA") >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 1
elif path.find("pkgB") >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 2
self.assertEquals(3, bitmap)
def test_getPackagePaths(self):
serviceName = 'service1'
manifestName = 'manifestA'
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
packagePaths = ManifestController.getPackagePaths(serviceName, manifestName);
print ">>>>>>>>>>>>" + str(packagePaths)
self.assertTrue(len(packagePaths) >= 2)
bitmap = 0
for path in packagePaths:
if path.find("pkgA") >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 1
elif path.find("pkgB") >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 2
self.assertEquals(3, bitmap)
def forceCleanUpDownloadedPkgs():
''' tries to cleanup packages from oldest date to latest ignoring packages downloaded in the last hour'''
import operator
import glob
import time
from agent.lib.packagemgr import PackageMgr
LOG.info('Force garbage collecting downloaded packages')
packageMount = pylons.config['agent_root']
lower_age_bound = float(pylons.config['packageMgr_package_min_age'])
cronus_pkgs = glob.glob(
os.path.join(PackageMgr.packagePath(), '*.cronus'))
file_list = []
for filename in cronus_pkgs:
try:
date_file_tuple = os.path.getatime(filename), os.path.getsize(
filename), filename
file_list.append(date_file_tuple)
except OSError, osErr:
LOG.error('Unable to get access time %s - %s' %
(filename, osErr))
def test_delete_cronus_package(self):
package = "pkgA-1.2.0.unix"
package_url = "http://repository.qa.ebay.com/cronus/test-data/agent/pkgA-1.2.0.unix.cronus"
package_path = os.path.join(PackageMgr.packagePath(), package + '.cronus')
# case 1 - package is not present
self.assertFalse(os.path.exists(package_path))
response = self.app.delete(url(controller='cleanup', package=package, action='deleteCronusPackage'), expect_errors = True)
self.assertEqual(response.status_int, 200)
# case 2 - package is present
body = json.dumps({'package': package_url, 'packageloc' : package + '.cronus'})
response = self.app.post(url(controller = 'distribution', action = 'startdownload', service = "dist"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertEqual(response.status_int, 200)
checkStatus(self, "http_download", response, 100)
self.assertTrue(os.path.exists(package_path))
response = self.app.delete(url(controller='cleanup', package=package, action='deleteCronusPackage'), expect_errors = True)
self.assertEqual(response.status_int, 200)
self.assertFalse(os.path.exists(package_path))
def test_same_pkg_download_parallel(self):
packages = ['http://repository.qa.ebay.com/cronus/test-data/agent/perlserver-1.0.0.unix.cronus']
manifest1 = 'bar'
manifest2 = 'blah'
service1 = 'foo'
service2 = 'lah'
try:
path = ServiceController.manifestPath(service1)
os.makedirs(path)
path = ServiceController.installedPkgPath(service1)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service1)
os.makedirs(path)
path = ServiceController.manifestPath(service2)
os.makedirs(path)
path = ServiceController.installedPkgPath(service2)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service2)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response1.status_int == 200, 'Manifest Post assert'
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response2.status_int == 200, 'Manifest Post assert'
checkStatus(self, 'create manifest bar', response1, timeout = 25)
checkStatus(self, 'create manifest baz', response2, timeout = 25)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('lah', 'blah'), 'perlserver')))
def test_cleanupPackages(self):
stopAgentGlobals()
# ok now let's create the directory structure of a bad service and packages
packagePath = PackageMgr.packagePath()
inProgPackage2 = os.path.join(packagePath, 'bar.inprogress')
inProgPackage1 = os.path.join(packagePath, 'foo.inprogress')
goodPackage1 = os.path.join(packagePath, 'baz')
goodPackage2 = os.path.join(packagePath, 'baz')
open(inProgPackage1, 'w').close()
open(inProgPackage2, 'w').close()
open(goodPackage1, 'w').close()
open(goodPackage2, 'w').close()
startAgentGlobals()
# we remove logic of inprogress file cleanup from packagemgr
# assert not os.path.exists(inProgPackage1)
# assert not os.path.exists(inProgPackage2)
assert os.path.exists(goodPackage1)
assert os.path.exists(goodPackage2)
def test_cleanupPackages(self):
stopAgentGlobals()
# ok now let's create the directory structure of a bad service and packages
packagePath = PackageMgr.packagePath()
inProgPackage2 = os.path.join(packagePath, 'bar.inprogress')
inProgPackage1 = os.path.join(packagePath, 'foo.inprogress')
goodPackage1 = os.path.join(packagePath, 'baz')
goodPackage2 = os.path.join(packagePath, 'baz')
open(inProgPackage1, 'w').close()
open(inProgPackage2, 'w').close()
open(goodPackage1, 'w').close()
open(goodPackage2, 'w').close()
startAgentGlobals()
# we remove logic of inprogress file cleanup from packagemgr
# assert not os.path.exists(inProgPackage1)
# assert not os.path.exists(inProgPackage2)
assert os.path.exists(goodPackage1)
assert os.path.exists(goodPackage2)
def test_package_min_age_during_forceDelete(self):
#import pdb;pdb.set_trace();
from agent.tests.unit.test_util import createManifest
serviceName = 'service5'
manifestName = 'manifestb'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
minage = pylons.config['packageMgr_package_min_age']
pylons.config['packageMgr_package_min_age'] = 86400
disk_threshold = pylons.config['health_disk_usage_percent_threshold']
pylons.config['health_disk_usage_percent_threshold'] = 1
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 4)
time.sleep(40)
self.assertEquals(len(os.listdir(pkgs)), 4)
pylons.config['packageMgr_package_age'] = age
pylons.config['packageMgr_package_min_age'] = minage
pylons.config['health_disk_usage_percent_threshold'] = disk_threshold
def parseUri(packageUri, path=None, packageloc=None):
""" static method to parse the uri
@throws - AgentException
"""
if (path == None):
from agent.lib.packagemgr import PackageMgr
path = PackageMgr.packagePath()
uri = urlparse.urlparse(packageUri)
if (uri.scheme != 'http'):
raise AgentException(
Errors.PACKAGE_SCHEME_ERROR,
'uri (%s) scheme(%s) not supported' % (packageUri, uri.scheme))
if (uri.path == ''):
raise AgentException(Errors.PACKAGE_PATH_ERROR,
'uri (%s) path is empty' % (packageUri))
# now parse the path. get the name and then verify that it matches the convention
if packageloc is not None:
# check if torrent download
if uri.path.endswith('.torrent'):
if not packageloc.endswith('.torrent'):
packageloc = packageloc + '.torrent'
packName = packageloc
else:
packName = uri.path.rpartition('/')[2]
match = PackageUtil.nameRe.match(packName)
if (match == None or match.group(0) != packName):
raise AgentException(
Errors.PACKAGE_PATH_ERROR,
'cannot find package name in path %s' % (uri.path))
# ok now we can fill out the dictionary
uriDict = {}
uriDict['uri'] = packageUri
uriDict['scheme'] = uri.scheme
uriDict['uripath'] = uri.path
uriDict['package'] = match.group(1)
uriDict['packageNameVersion'] = match.group(2)
uriDict['inProgressPackage'] = PackageUtil.inProgressPath(
uriDict['package'])
uriDict['packageName'] = match.group(3)
uriDict['propName'] = uriDict['package'] + '.prop'
uriDict['packageVersion'] = match.group(4)
uriDict['packageVersionMajor'] = match.group(5)
uriDict['packageVersionMinor'] = match.group(6)
uriDict['packageVersionBuild'] = match.group(7)
uriDict['packagePlat'] = match.group(8)
# path specific attributes
uriDict['packagePath'] = os.path.join(path, uriDict['package'])
uriDict['inProgressPackagePath'] = os.path.join(
path, uriDict['inProgressPackage'])
uriDict['propPath'] = os.path.join(path, uriDict['propName'])
# check if bittorrent transfer and define the necessary keys
if match.group(9) is not None and match.group(9) == '.torrent':
uriDict['scheme'] = 'bittorrent'
uriDict['torrentName'] = uriDict['package'] + '.torrent'
uriDict['torrentPath'] = os.path.join(path, uriDict['torrentName'])
# calculate prop url
# append path with .prop - mons: leaving the below code in place in case we support other protocols in future
uriScheme = uri.scheme
if (uriScheme !=
"http"): #only use http to download .prop and .torrent files
uriScheme = "http"
if uri.path.endswith('.torrent'):
uripath = uri.path.strip('.torrent')
else:
uripath = uri.path
propParseResult = urlparse.ParseResult(uriScheme, uri.netloc,
uripath + '.prop', uri.params,
uri.query, uri.fragment)
propUri = urlparse.urlunparse(propParseResult)
uriDict['propUri'] = propUri
# calculate the torrent url
# append path with .torrent
torrentParseResult = urlparse.ParseResult(uriScheme, uri.netloc,
uripath + '.torrent',
uri.params, uri.query,
uri.fragment)
torrentUri = urlparse.urlunparse(torrentParseResult)
uriDict['torrentUri'] = torrentUri
return uriDict
def testDelete(self):
def deleteTestDir(service):
LOG.debug('************ service = %s' % service)
response = self.app.delete(url(controller='service', service=service, action='delete'), expect_errors = True)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
# assert response.status_int == 500, "HTTP response != 500"
def makeTestDir(path):
os.makedirs(path)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'installed-packages'))
def createManifests(mf_path):
os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
latest = os.path.join(mf_path, 'm3.0')
os.makedirs(os.path.join(latest, 'dummy_dir3.0'))
utils.symlink(latest, os.path.join(mf_path, 'active'))
return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')
def makePackageContent(path, pkgPath, pkgPropPath):
pkgFile = file(pkgPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
pkgFile = file(pkgPropPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(path, uname)
def createTestThread(serviceName):
appGlobal = config['pylons.app_globals']
testTh = WaitThread(appGlobal.threadMgr, ServiceController.serviceCat(serviceName))
testTh.start()
return testTh
def startTestProcess():
cmd = utils.sudoCmd(["sleep", "5"], pylons.config['app_user_account'])
return Popen(cmd)
path1 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'foo')
path2 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'bar')
path3 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'agent')
deleteTestDir('foo')
deleteTestDir('bar')
deleteTestDir('agent')
# make dirs
makeTestDir(path1)
makeTestDir(path2)
makeTestDir(path3)
all_mf, active_mf = createManifests(ServiceController.manifestPath('agent'))
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(ServiceController.serviceRootPath(), uname)
pkgDir = PackageMgr.packagePath()
pkgPath = os.path.join(pkgDir, "foo.cronus")
pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop")
makePackageContent(pkgDir, pkgPath, pkgPropPath)
# create threads
testThFoo = createTestThread('foo')
testThBar = createTestThread('bar')
testThAgent = createTestThread('agent')
# start process
process = startTestProcess()
# start testing
LOG.debug('************ start cleanup')
response = self.app.post(url(controller='cleanup', action='post'))
LOG.debug ('Delete response body = ' + response.body)
body = json.loads(response.body)
tm = time.time()
while (tm + 10 > time.time()):
response = self.app.get(body['status'], expect_errors = True)
LOG.debug ('Status response body = ' + response.body)
body = json.loads(response.body)
print body
if (body['progress'] == 100):
break
time.sleep(0.1)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
time.sleep(0.1)
assert not os.path.exists(path1), 'service foo does exist or is not a directory'
assert not os.path.exists(path2), 'service bar does exist or is not a directory'
assert os.path.exists(path3), 'service agent does NOT exist or is not a directory'
assert not testThFoo.isAlive(), 'thread Foo is still alive'
assert not testThBar.isAlive(), 'thread Bar is still alive'
assert not testThAgent.isAlive(), 'thread Agent is still alive'
assert not os.path.exists(pkgPath), 'package foo exists'
assert not os.path.exists(pkgPropPath), 'package prop foo exists'
assert os.path.exists(pkgDir), 'package directory does not exist'
# ensure agent cleanup is proper
active_mf_path = manifestutil.manifestPath('agent', active_mf)
active_link = os.path.join(ServiceController.manifestPath('agent'), 'active')
all_mf.remove(active_mf)
actual_active_mf_path = utils.readlink(active_link)
self.assertTrue(os.path.exists(active_mf_path), 'active agent manifest got deleted but shouldn\t have')
self.assertTrue(os.path.exists(active_link), 'agent active link missing')
self.assertEqual(active_mf_path, actual_active_mf_path, 'agent active link pointing to some wrong manifest; link broken?')
for mf in all_mf:
agnt_mf_path = manifestutil.manifestPath('agent', mf)
self.assertFalse(os.path.exists(agnt_mf_path), 'non active agent mf %s should have been deleted' % mf)
def test_parsing_garbage(self):
# test the url parsing
try:
PackageUtil.parseUri('asdfawert892375[jrgjhnjbuy43w5897t^!R$#W_R(@$^(_#$(', PackageMgr.packagePath())
assert False, 'parse garbage'
except AgentException:
pass
def parseUri(packageUri, path = None, packageloc = None):
""" static method to parse the uri
@throws - AgentException
"""
if (path == None):
from agent.lib.packagemgr import PackageMgr
path = PackageMgr.packagePath()
uri = urlparse.urlparse(packageUri)
if (uri.scheme != 'http'):
raise AgentException(Errors.PACKAGE_SCHEME_ERROR,
'uri (%s) scheme(%s) not supported' % (packageUri, uri.scheme))
if (uri.path == ''):
raise AgentException(Errors.PACKAGE_PATH_ERROR,
'uri (%s) path is empty' % (packageUri))
# now parse the path. get the name and then verify that it matches the convention
if packageloc is not None:
# check if torrent download
if uri.path.endswith('.torrent'):
if not packageloc.endswith('.torrent'):
packageloc = packageloc + '.torrent'
packName = packageloc
else:
packName = uri.path.rpartition('/')[2]
match = PackageUtil.nameRe.match(packName)
if (match == None or match.group(0) != packName):
raise AgentException(Errors.PACKAGE_PATH_ERROR,
'cannot find package name in path %s' % (uri.path))
# ok now we can fill out the dictionary
uriDict = {}
uriDict['uri'] = packageUri
uriDict['scheme'] = uri.scheme
uriDict['uripath'] = uri.path
uriDict['package'] = match.group(1)
uriDict['packageNameVersion'] = match.group(2)
uriDict['inProgressPackage'] = PackageUtil.inProgressPath(uriDict['package'])
uriDict['packageName'] = match.group(3)
uriDict['propName'] = uriDict['package'] + '.prop'
uriDict['packageVersion'] = match.group(4)
uriDict['packageVersionMajor'] = match.group(5)
uriDict['packageVersionMinor'] = match.group(6)
uriDict['packageVersionBuild'] = match.group(7)
uriDict['packagePlat'] = match.group(8)
# path specific attributes
uriDict['packagePath'] = os.path.join(path, uriDict['package'])
uriDict['inProgressPackagePath'] = os.path.join(path, uriDict['inProgressPackage'])
uriDict['propPath'] = os.path.join(path, uriDict['propName'])
# check if bittorrent transfer and define the necessary keys
if match.group(9) is not None and match.group(9) == '.torrent':
uriDict['scheme'] = 'bittorrent'
uriDict['torrentName'] = uriDict['package'] + '.torrent'
uriDict['torrentPath'] = os.path.join(path, uriDict['torrentName'])
# calculate prop url
# append path with .prop - mons: leaving the below code in place in case we support other protocols in future
uriScheme = uri.scheme
if (uriScheme != "http"): #only use http to download .prop and .torrent files
uriScheme = "http"
if uri.path.endswith('.torrent'):
uripath = uri.path.strip('.torrent')
else:
uripath = uri.path
propParseResult = urlparse.ParseResult(uriScheme, uri.netloc, uripath + '.prop',
uri.params, uri.query, uri.fragment)
propUri = urlparse.urlunparse(propParseResult)
uriDict['propUri'] = propUri
# calculate the torrent url
# append path with .torrent
torrentParseResult = urlparse.ParseResult(uriScheme, uri.netloc, uripath + '.torrent',
uri.params, uri.query, uri.fragment)
torrentUri = urlparse.urlunparse(torrentParseResult)
uriDict['torrentUri'] = torrentUri
return uriDict
def __deletePackages(self):
""" delete all packages from pakcage directory """
package_path = PackageMgr.packagePath()
ServiceDelete.deleteFolderContents(package_path, onlyChildren=True)
def testDelete(self):
def deleteTestDir(service):
LOG.debug('************ service = %s' % service)
response = self.app.delete(url(controller='service',
service=service,
action='delete'),
expect_errors=True)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
# assert response.status_int == 500, "HTTP response != 500"
def makeTestDir(path):
os.makedirs(path)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'installed-packages'))
def createManifests(mf_path):
os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
latest = os.path.join(mf_path, 'm3.0')
os.makedirs(os.path.join(latest, 'dummy_dir3.0'))
utils.symlink(latest, os.path.join(mf_path, 'active'))
return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')
def makePackageContent(path, pkgPath, pkgPropPath):
pkgFile = file(pkgPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
pkgFile = file(pkgPropPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(path, uname)
def createTestThread(serviceName):
appGlobal = config['pylons.app_globals']
testTh = WaitThread(appGlobal.threadMgr,
ServiceController.serviceCat(serviceName))
testTh.start()
return testTh
def startTestProcess():
cmd = utils.sudoCmd(["sleep", "5"],
pylons.config['app_user_account'])
return Popen(cmd)
if os.name == 'nt':
LOG.warning(
'Services cleanup not supported on windows. Skipping test...')
return
path1 = os.path.join(pylons.config['agent_root'], 'service_nodes',
'foo')
path2 = os.path.join(pylons.config['agent_root'], 'service_nodes',
'bar')
path3 = os.path.join(pylons.config['agent_root'], 'service_nodes',
'agent')
deleteTestDir('foo')
deleteTestDir('bar')
deleteTestDir('agent')
# make dirs
makeTestDir(path1)
makeTestDir(path2)
makeTestDir(path3)
all_mf, active_mf = createManifests(
ServiceController.manifestPath('agent'))
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(ServiceController.serviceRootPath(),
uname)
pkgDir = PackageMgr.packagePath()
pkgPath = os.path.join(pkgDir, "foo.cronus")
pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop")
makePackageContent(pkgDir, pkgPath, pkgPropPath)
# create threads
testThFoo = createTestThread('foo')
testThBar = createTestThread('bar')
testThAgent = createTestThread('agent')
# start process
process = startTestProcess()
# start testing
LOG.debug('************ start cleanup')
response = self.app.post(url(controller='cleanup', action='post'))
LOG.debug('Delete response body = ' + response.body)
body = json.loads(response.body)
tm = time.time()
while (tm + 10 > time.time()):
response = self.app.get(body['status'], expect_errors=True)
LOG.debug('Status response body = ' + response.body)
body = json.loads(response.body)
print body
if (body['progress'] == 100):
break
time.sleep(0.1)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
time.sleep(0.1)
assert not os.path.exists(
path1), 'service foo does exist or is not a directory'
assert not os.path.exists(
path2), 'service bar does exist or is not a directory'
assert os.path.exists(
path3), 'service agent does NOT exist or is not a directory'
assert not testThFoo.isAlive(), 'thread Foo is still alive'
assert not testThBar.isAlive(), 'thread Bar is still alive'
assert not testThAgent.isAlive(), 'thread Agent is still alive'
assert not os.path.exists(pkgPath), 'package foo exists'
assert not os.path.exists(pkgPropPath), 'package prop foo exists'
assert os.path.exists(pkgDir), 'package directory does not exist'
# ensure agent cleanup is proper
active_mf_path = ManifestController.manifestPath('agent', active_mf)
active_link = os.path.join(ServiceController.manifestPath('agent'),
'active')
all_mf.remove(active_mf)
actual_active_mf_path = utils.readlink(active_link)
self.assertTrue(
os.path.exists(active_mf_path),
'active agent manifest got deleted but shouldn\t have')
self.assertTrue(os.path.exists(active_link),
'agent active link missing')
self.assertEqual(
active_mf_path, actual_active_mf_path,
'agent active link pointing to some wrong manifest; link broken?')
for mf in all_mf:
agnt_mf_path = ManifestController.manifestPath('agent', mf)
self.assertFalse(
os.path.exists(agnt_mf_path),
'non active agent mf %s should have been deleted' % mf)
def __deletePackages(self):
""" delete all packages from pakcage directory """
package_path = PackageMgr.packagePath()
ServiceDelete.deleteFolderContents(package_path, onlyChildren = True)