def __deleteServices(self):
""" delete all services except agent itself; clear all manifests in 'agent' itself except current active"""
self._updateStatus(progress = 60)
services = ServiceController.getServices()
#kill all service threads
self._threadMgr.stopServiceThread()
#remove folder
for service in services:
if 'agent' == service:
self.__delAllExceptActiveManifests(service)
continue
path = ServiceController.servicePath(service)
# retry service cleanup
for _ in range(3):
if not os.path.exists(path):
break
ServiceDelete.deleteFolderContents(path)
# sleep here a bit to ensure delete is complete
time.sleep(1)
if os.path.exists(path):
msg = 'Could not delete service %s completely even after 3 retries.' % service
LOG.error(msg)
raise Exception(msg)
self._updateStatus(progress = 90)
def test_multiple_manifest_create(self):
""" when a manifest creation is in progress for a service, another creation should block """
packages = ["http://www.stackscaling.com/agentrepo/pkgA-1.2.0.unix.cronus"]
service = 'foo'
manifest1 = 'bar'
manifest2 = 'car'
try:
for pkg in packages:
mockDownloadPkg(pkg)
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
self.assertEquals(response1.status_int, 200, 'Manifest1 Post assert - should go through')
try:
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
self.assertFalse(True, 'Expected an exception but did not get one!')
except AppError:
pass
checkStatus(self, 'create manifest bar', response1, timeout = 25)
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'pkgA')))
self.assertFalse(islink(os.path.join(manifestutil.manifestPath('foo', 'car'), 'pkgA')))
def test_post_manifest_inprogress_errorout(self):
packages = ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']
manifest = 'bar'
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response.status_int == 200, 'Manifest Post assert'
try:
response = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
#should error out!
raise Exception('Oops! Expected AppError but did not get any')
except AppError:
pass
def testDelete2(self):
createManifest(self)
# need to change the owner of the service dir to be cronus
servicePath = ServiceController.servicePath('foo')
if (not os.name == 'nt'):
import pwd
uname = pylons.config['agent_user_account']
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
rchown(servicePath, uid, gid)
response = self.app.delete(
url(controller='service', service='foo', action='delete'))
assert response.status_int == 200, "HTTP response != 200"
body = json.loads(response.body)
tm = time.time()
while (tm + 5 > time.time()):
response = self.app.get(body['status'], expect_errors=True)
LOG.debug('************* Status response body = ' + response.body)
body = json.loads(response.body)
if (body['progress'] == 100):
break
assert not os.path.isdir(ServiceController.servicePath('foo'))
def cleanupOrphanedPackages(checkAge = False):
''' API to cleanup Orphaned Packages For All Services '''
from agent.controllers.service import ServiceController
#services = os.listdir(service_nodes)
from agent.controllers.manifest import ManifestController
#import pdb;pdb.set_trace()
services = ServiceController.getServices()
service_nodes = ServiceController.serviceRootPath()
linkedPaths = []
sysServices = ['agent']
LOG.info('Garbage collecting orphaned installed packages')
for service in services:
try:
if service in sysServices:
LOG.debug('system services cannot be garbage collected')
else:
servicePath = os.path.join(service_nodes, service)
installedPkgPath = os.path.join(servicePath, 'installed-packages')
linkedPaths.extend(ManifestController.getAllSymLinks(service))
linkedPaths.extend(manifestutil.getModuleSymLinks(service))
LOG.debug('symLinks returned %s' % linkedPaths)
installedPkgPaths = PackageUtil.getAllInstalledPackages(installedPkgPath)
LOG.debug('installedPkgPaths returned for the service %s' % installedPkgPaths)
if len(installedPkgPaths) > 0:
orphanPkgs = set(installedPkgPaths) - set(linkedPaths)
LOG.debug('orphanPkgs returned %s' % orphanPkgs)
PackageUtil.cleanupInstalledPkgs(installedPkgPath, orphanPkgs)
except BaseException as excep:
LOG.error('Failed to proceed with garbage collection %s' % str(excep))
# agent-804, manifests only contains folders, need to delete if file is in manifests
servicePath = os.path.join(service_nodes, service)
if not os.path.isdir(servicePath):
utils.runsyscmd('rm -f %s' % servicePath)
LOG.info('Garbage collecting orphaned installed packages completed')
def test_multiple_manifest_create(self):
""" when a manifest creation is in progress for a service, another creation should block """
packages = ["http://repository.qa.ebay.com/cronus/test-data/agent/pkgA-1.2.0.unix.cronus"]
service = 'foo'
manifest1 = 'bar'
manifest2 = 'car'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
self.assertEquals(response1.status_int, 200, 'Manifest1 Post assert - should go through')
try:
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
self.assertFalse(True, 'Expected an exception but did not get one!')
except AppError:
pass
checkStatus(self, 'create manifest bar', response1, timeout = 25)
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'pkgA')))
self.assertFalse(islink(os.path.join(ManifestController.manifestPath('foo', 'car'), 'pkgA')))
def __deleteServices(self):
""" delete all services except agent itself; clear all manifests in 'agent' itself except current active"""
self._updateStatus(progress=60)
services = ServiceController.getServices()
#kill all service threads
self._threadMgr.stopServiceThread()
#remove folder
for service in services:
if 'agent' == service:
self.__delAllExceptActiveManifests(service)
continue
path = ServiceController.servicePath(service)
# retry service cleanup
for _ in range(3):
if not os.path.exists(path):
break
ServiceDelete.deleteFolderContents(path)
# sleep here a bit to ensure delete is complete
time.sleep(1)
if os.path.exists(path):
msg = 'Could not delete service %s completely even after 3 retries.' % service
LOG.error(msg)
raise Exception(msg)
self._updateStatus(progress=90)
def test_post_manifest_inprogress_ok(self):
service = 'foo'
manifest = 'blahblah'
try:
path = ServiceController.servicePath(service)
if os.path.exists(path):
if os.name == 'nt':
cmd = 'rm -r %s' % path
LOG.debug("running command %s" % cmd)
os.system(cmd)
else:
shutil.rmtree(path)
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(service, manifest))
os.makedirs(inProgressPath)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus"], manifest = 'blahblah', createDirs = False)
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'blahblah'), 'pkgA')))
def test_post_manifest_inprogress_errorout(self):
packages = ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']
manifest = 'bar'
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response.status_int == 200, 'Manifest Post assert'
try:
response = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
#should error out!
raise Exception('Oops! Expected AppError but did not get any')
except AppError:
pass
def test_cleanupOrphanedPackages2(self):
from agent.lib.utils import rchown
from agent.controllers.service import ServiceController
from agent.controllers.manifest import ManifestController
from agent.tests.unit.test_util import createManifest
if os.name == 'nt':
pass
else:
serviceName = 'service2'
manifestName = 'manifestA'
serviceName1 = 'service3'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"
],
service=serviceName1,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
installedPkgPath = os.path.realpath(
os.path.join(ServiceController.serviceRootPath(), serviceName,
'installed-packages'))
installedPkgPath1 = os.path.realpath(
os.path.join(ServiceController.serviceRootPath(), serviceName1,
'installed-packages'))
if (not os.name == 'nt'):
import pwd
uname = pylons.config['agent_user_account']
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
rchown(installedPkgPath, uid, gid)
rchown(installedPkgPath1, uid, gid)
self.assertEquals(len(os.listdir(installedPkgPath)), 1)
self.assertEquals(len(os.listdir(installedPkgPath1)), 1)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName1,
manifest=manifestName),
expect_errors=True)
pylons.config['packageMgr_install_package_age'] = 0
pylons.config['packageMgr_install_package_min_age'] = 0.0
PackageUtil.cleanupOrphanedPackages()
self.assertEquals(len(os.listdir(installedPkgPath)), 0)
self.assertEquals(len(os.listdir(installedPkgPath1)), 0)
def createManifest(testController,
packages = ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus'],
manifest = 'bar', service = 'foo', createDirs = True):
try:
path = ServiceController.servicePath(service)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'modules'))
os.makedirs(os.path.join(path, '.appdata'))
os.makedirs(os.path.join(path, '.data'))
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
try:
for pkgUri in packages:
mockDownloadPkg(pkgUri)
# pkgDict = PackageUtil.parseUri(pkgUri)
# pkgName = pkgDict['package']
# localPkgRoot = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'packages')
# src = os.path.join(localPkgRoot, pkgName)
# shutil.copy(src, pkgDict['packagePath'])
# propPkgName = pkgDict['propName']
# propSrc = os.path.join(localPkgRoot, propPkgName)
# shutil.copy(propSrc, pkgDict['propPath'])
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response = testController.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response.status_int == 200, 'Manifest Post assert'
LOG.debug('response body = %s' % response.body)
body = json.loads(response.body)
# wait until the status is done
tm = time.time()
now = tm
while (tm + 120 > now):
response = testController.app.get(body['status'])
body = json.loads(response.body)
LOG.debug("createManifest ********** progress = %s" % body['progress'])
if (int(body['progress']) == 100):
break
time.sleep(0.1)
now = time.time()
assert tm + 120 >= now, 'Create manifest timed out'
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
LOG.debug ('Status response body = %s' % str(body))
assert int(body['progress']) == 100
def stopServiceThread(self):
""" stop all service threads """
from agent.controllers.service import ServiceController
LOG.info("Stopping service thread")
with self.__lock:
services = ServiceController.getServices()
for service in services:
threads = self.getThreadByCat(ServiceController.serviceCat(service), fastbreak = False)
for thread in threads:
thread.stop()
def stopServiceThread(self):
""" stop all service threads """
from agent.controllers.service import ServiceController
LOG.info("Stopping service thread")
with self.__lock:
services = ServiceController.getServices()
for service in services:
threads = self.getThreadByCat(ServiceController.serviceCat(service), fastbreak=False)
for thread in threads:
thread.stop()
def getManifests(service):
""" return the list of manifests under a specific service """
from agent.lib.agent_thread.manifest_create import ManifestCreate
manifests = []
dirContent = os.listdir(ServiceController.manifestPath(service))
for item in dirContent:
path = os.path.join(ServiceController.manifestPath(service), item)
if (os.path.isdir(path) and not ManifestCreate.isInProgress(path)):
if (not islink(path)):
manifests.append(item)
return sorted(manifests)
def getManifests(service):
""" return the list of manifests under a specific service """
from agent.lib.agent_thread.manifest_create import ManifestCreate
manifests = []
dirContent = os.listdir(ServiceController.manifestPath(service))
for item in dirContent:
path = os.path.join(ServiceController.manifestPath(service), item)
if (os.path.isdir(path) and not ManifestCreate.isInProgress(path)):
if (not islink(path)):
manifests.append(item)
return sorted(manifests)
def test_delete_active_manifest(self):
createManifest(self)
self.assertTrue(os.path.isdir(ServiceController.manifestPath('foo')))
currentPath = os.getcwd()
os.chdir(ServiceController.manifestPath('foo'))
manifestPath = 'bar' #ManifestController.manifestPath('foo', 'bar'); use short manifest name instead of full path
symlink(manifestPath, 'active')
os.chdir(currentPath)
response = self.app.delete(url(controller = 'manifest', action = 'delete', service = "foo", manifest = "bar"),
expect_errors = True)
self.assertEquals(500, response.status_int)
body = json.loads(response.body)
self.assertEquals(Errors.MANIFEST_DELETING_ACTIVE_MANIFEST, body['error'])
def activateManifest(testController, manifest = 'bar', service = 'foo'):
body = json.dumps({'manifest':manifest})
response = testController.app.post(url(controller = 'manifest', action = 'activate', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response.status_int == 200, 'Action get assert'
body = json.loads(response.body)
tm = time.time()
while (tm + 120 > time.time()):
response = testController.app.get(body['status'])
body = json.loads(response.body)
LOG.debug("activateManifest ********** progress = %s" % body['progress'])
if (int(body['progress']) == 100):
break
time.sleep(0.1)
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
LOG.debug ('Status response body = %s' % str(body))
assert body['progress'] == 100
# let's make sure the link is there correctly
activePath = os.path.join(ServiceController.manifestPath(service), 'active')
LOG.debug ('active path = ' + activePath)
assert islink(activePath)
link = readlink(activePath)
LOG.debug ('link = ' + link)
assert link == manifest
def manifestPath(service, manifest):
"""
compute the path to this manifest
@param service: name of the service
@param manifest: name of the manifest
"""
return os.path.join(ServiceController.manifestPath(service), manifest)
def test_getAllSymLinks(self):
#import pdb;
#pdb.set_trace()
serviceName = '.sbe.appService.SI1'
manifestName = 'manifestA'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf'
symLinks = ManifestController.getAllSymLinks(serviceName)
""" return all the symlinks from manifests to packages for a given service"""
LOG.debug('calling getAllSymLinks')
#serviceName = '.sbe.appService.SI1'
linkedPaths = []
#pdb.set_trace()
#manPath = '/ebay/srengarajan/e2e/local-deploy/service_nodes/.sbe.appService.SI1/manifests'
installedPkgPath = os.path.join(ServiceController.servicePath(serviceName), 'installed-packages')
pathToPkgA = os.path.join(installedPkgPath, 'pkgA')
pathToPkgA120 = os.path.join(pathToPkgA, '1.2.0.unix')
pathToPkgB = os.path.join(installedPkgPath, 'pkgB')
pathToPkgB060 = os.path.join(pathToPkgB, '0.6.0.unix')
bitmap = 0
for path in symLinks:
if path.find(pathToPkgA120) >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 1
elif path.find(pathToPkgB060) >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 2
self.assertEquals(3, bitmap)
def __init__(self, threadMgr, service, manifest):
""" Constructor """
from agent.controllers.service import ServiceController
AgentThread.__init__(self, threadMgr, cat = [ServiceController.serviceCat(service)], name = 'delete_manifest')
self.service = service
self.manifest = manifest
def delete(self, service, manifest):
""" Delete a new service object """
try:
path = ManifestController.manifestPath(service, manifest)
if (not os.path.isdir(path)):
return errorResult(request, response, Errors.MANIFEST_NOT_FOUND,
'manifest (%s/%s) missing service' % (service, manifest),
controller = self)
# first check that this isn't the active manifest
path = os.path.join(ServiceController.manifestPath(service), 'active')
if (os.path.exists(path)):
activePath = os.path.basename(readlink(path))
deletePath = os.path.basename(ManifestController.manifestPath(service, manifest))
if (activePath == deletePath):
return errorResult(request, response, Errors.MANIFEST_DELETING_ACTIVE_MANIFEST,
'Manifest(%s, %s) attempting to delete active manifest'
% (service, manifest),
controller = self)
# now try to delete the manifest directory
appGlobal = config['pylons.app_globals']
manThread = ManifestDelete(appGlobal.threadMgr, service, manifest)
self.injectJobCtx(manThread)
manThread.start()
manThread.threadMgrEvent.wait()
return statusResult(request, response, manThread, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error for delete manifest(%s/%s) - %s - %s' %
(service, manifest, str(excep), traceback.format_exc(2)),
controller = self)
def manifestPath(service, manifest):
"""
compute the path to this manifest
@param service: name of the service
@param manifest: name of the manifest
"""
return os.path.join(ServiceController.manifestPath(service), manifest)
def test_getAllSymLinks(self):
#import pdb;
#pdb.set_trace()
serviceName = '.sbe.appService.SI1'
manifestName = 'manifestA'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf'
symLinks = manifestutil.getAllSymLinks(serviceName)
""" return all the symlinks from manifests to packages for a given service"""
LOG.debug('calling getAllSymLinks')
linkedPaths = []
installedPkgPath = os.path.join(ServiceController.servicePath(serviceName), 'installed-packages')
pathToPkgA = os.path.join(installedPkgPath, 'pkgA')
pathToPkgA120 = os.path.join(pathToPkgA, '1.2.0.unix')
pathToPkgB = os.path.join(installedPkgPath, 'pkgB')
pathToPkgB060 = os.path.join(pathToPkgB, '0.6.0.unix')
bitmap = 0
for path in symLinks:
if path.find(pathToPkgA120) >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 1
elif path.find(pathToPkgB060) >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 2
self.assertEquals(3, bitmap)
def test_delete_active_manifest(self):
createManifest(self)
self.assertTrue(os.path.isdir(ServiceController.manifestPath('foo')))
currentPath = os.getcwd()
os.chdir(ServiceController.manifestPath('foo'))
manifestPath = 'bar' #ManifestController.manifestPath('foo', 'bar'); use short manifest name instead of full path
symlink(manifestPath, 'active')
os.chdir(currentPath)
response = self.app.delete(url(controller = 'manifest', action = 'delete', service = "foo", manifest = "bar"),
expect_errors = True)
self.assertEquals(500, response.status_int)
body = json.loads(response.body)
self.assertEquals(Errors.MANIFEST_DELETING_ACTIVE_MANIFEST, body['error'])
def getAllSymLinks(service):
""" return all the symlinks from manifests to packages for a given service"""
LOG.debug('calling getAllSymLinks %s' % service)
linkedPaths = []
manPath = ServiceController.manifestPath(service)
LOG.debug('manifestPath is %s' % manPath)
for path in os.listdir(manPath):
LOG.debug('path is %s' % path)
if not islink(os.path.join(manPath, path)):
LOG.debug('path is dir not a link')
pkgPaths = [
packageDir
for packageDir in os.listdir(os.path.join(manPath, path))
]
LOG.debug('pkgPaths is %s' % pkgPaths)
for pkgPath in pkgPaths:
try:
LOG.debug('pkgPath is %s' % pkgPath)
if not os.path.isfile(
os.path.join(
manPath, path, pkgPath)) and islink(
os.path.join(manPath, path, pkgPath)):
LOG.debug('pkgPaths is %s' % pkgPath)
targetPath = os.path.abspath(
readlink(os.path.join(manPath, path, pkgPath)))
linkedPaths.append(targetPath)
LOG.debug('targetPath is %s ' % targetPath)
except BaseException as exc:
LOG.error('failed to read link for the pkg path %s' %
str(exc))
return linkedPaths
def test_post2(self):
# successful post
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def __init__(self, threadMgr, service, manifest, packages, attemptDownload = True, forcePackages = None, skipProp = True):
""" Constructor """
DownloadHelper.__init__(self, threadMgr, cat = [ServiceController.serviceCat(service)], name = 'create_manifest')
self.__manifest = manifest
self.__service = service
self.__packages = packages
self.__attemptDownload = attemptDownload
self.__forcePackages = forcePackages
self.__skipProp = skipProp
def __init__(self, threadMgr, service, manifest):
""" Constructor """
from agent.controllers.service import ServiceController
AgentThread.__init__(self,
threadMgr,
cat=[ServiceController.serviceCat(service)],
name='delete_manifest')
self.service = service
self.manifest = manifest
def test_inprogress_pkg_download(self):
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress')
inprogressFile = open(inprogressPath, 'w')
inprogressFile.write('somegarbage')
inprogressFile.close()
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_inprogress_pkg_download(self):
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress')
inprogressFile = open(inprogressPath, 'w')
inprogressFile.write('somegarbage')
inprogressFile.close()
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
def activateManifest(testController, manifest='bar', service='foo'):
body = json.dumps({'manifest': manifest})
response = testController.app.post(
url(controller='action', action='activatemanifest', service=service),
headers={'Content-Type': 'application/json'},
params=body)
assert response.status_int == 200, 'Action get assert'
body = json.loads(response.body)
tm = time.time()
while (tm + 120 > time.time()):
response = testController.app.get(body['status'])
body = json.loads(response.body)
LOG.debug("activateManifest ********** progress = %s" %
body['progress'])
if (int(body['progress']) == 100):
break
time.sleep(0.1)
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
LOG.debug('Status response body = %s' % str(body))
assert body['progress'] == 100
# let's make sure the link is there correctly
activePath = os.path.join(ServiceController.manifestPath(service),
'active')
LOG.debug('active path = ' + activePath)
assert islink(activePath)
link = readlink(activePath)
LOG.debug('link = ' + link)
if os.name == 'nt':
manifestPath = os.path.join(ServiceController.manifestPath(service),
manifest)
assert (link == manifestPath or link == manifest)
else:
assert link == manifest
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
try:
activePath = os.path.join(
ServiceController.manifestPath(self._service), 'active')
oldManifest = None
# make sure that if the active path exists, it's a link
# if not log that and delete the link
if (os.path.exists(activePath) and not os.name == 'nt'
and not islink(activePath)):
self.__LOG.error('%s is not a link. Attempted to delete' %
activePath)
shutil.rmtree(activePath)
if (os.path.exists(activePath)):
oldManifest = os.path.basename(readlink(activePath))
else:
raise AgentException(
error=Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
errorMsg='No active manifest - cannot deactivate service')
self.__deactivateManifest(self._service, oldManifest)
self.__removeSymlink(self._service)
except SystemExit as exc:
failed = True
if (len(exc.args) == 2):
# ok we got {err code, err msg}
errorCode = exc.args[0]
errorMsg = exc.args[1]
raise exc
except AgentException as exc:
failed = True
errorMsg = 'Deactivate Manifest - Agent Exception - %s' % exc.getMsg(
)
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Deactivate Manifest - Unknown error - (%s) - %s - %s' \
% (self._service, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.warning(errorMsg)
self._updateStatus(httpStatus=500,
error=errorCode,
errorMsg=errorMsg)
self.__LOG.debug('Done: activate manifest for (%s)' %
(self._service))
self._updateProgress(100)
def getActiveManifestPath(service):
"""
return the name of the active manifest under a specific service
@param service: name of service
@return: path of active manifest, or empty string if no active manifest
"""
activePath = os.path.join(ServiceController.manifestPath(service), 'active')
if (not os.path.exists(activePath)):
return ''
return readlink(activePath)
def cleanupOrphanedPackages(checkAge=False):
''' API to cleanup Orphaned Packages For All Services '''
from agent.controllers.service import ServiceController
#services = os.listdir(service_nodes)
from agent.controllers.manifest import ManifestController
#import pdb;pdb.set_trace()
services = ServiceController.getServices()
service_nodes = ServiceController.serviceRootPath()
linkedPaths = []
sysServices = ['agent']
LOG.info('Garbage collecting orphaned installed packages')
for service in services:
try:
if service in sysServices:
LOG.debug('system services cannot be garbage collected')
else:
servicePath = os.path.join(service_nodes, service)
installedPkgPath = os.path.join(servicePath,
'installed-packages')
linkedPaths.extend(
ManifestController.getAllSymLinks(service))
linkedPaths.extend(manifestutil.getModuleSymLinks(service))
LOG.debug('symLinks returned %s' % linkedPaths)
installedPkgPaths = PackageUtil.getAllInstalledPackages(
installedPkgPath)
LOG.debug('installedPkgPaths returned for the service %s' %
installedPkgPaths)
if len(installedPkgPaths) > 0:
orphanPkgs = set(installedPkgPaths) - set(linkedPaths)
LOG.debug('orphanPkgs returned %s' % orphanPkgs)
PackageUtil.cleanupInstalledPkgs(
installedPkgPath, orphanPkgs)
except BaseException as excep:
LOG.error('Failed to proceed with garbage collection %s' %
str(excep))
# agent-804, manifests only contains folders, need to delete if file is in manifests
servicePath = os.path.join(service_nodes, service)
if not os.path.isdir(servicePath):
utils.runsyscmd('rm -f %s' % servicePath)
LOG.info('Garbage collecting orphaned installed packages completed')
def getActiveManifestPath(service):
"""
return the name of the active manifest under a specific service
@param service: name of service
@return: path of active manifest, or empty string if no active manifest
"""
activePath = os.path.join(ServiceController.manifestPath(service),
'active')
if (not os.path.exists(activePath)):
return ''
return readlink(activePath)
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
activeManifest = None
try:
activePath = os.path.join(
ServiceController.manifestPath(self._service), 'active')
# make sure that the active path exists and it is a link
# Should we check this again since we already have a check in action controller
if not os.path.exists(activePath) or not islink(activePath):
raise AgentException(
error=Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
errorMsg='No active manifest - cannot reset service')
activeManifest = os.path.basename(readlink(activePath))
self.__shutdownManifest(self._service, activeManifest)
self.__deactivateManifest(self._service, activeManifest)
self.__activateManifest(self._service, activeManifest)
self.__startupManifest(self._service, activeManifest)
self.__LOG.info('Done: reset service for (%s/%s)' %
(self._service, activeManifest))
self._updateStatus(progress=100)
except AgentException as exc:
failed = True
errorMsg = 'Activate Manifest - Agent Exception - %s' % exc.getMsg(
)
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Activate Manifest - Unknown error - (%s/%s) - %s - %s' \
% (self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.error(errorMsg)
if self._service and activeManifest:
try:
self.__LOG.info(
'Reset service %s failed, shutdown to cleanup' %
self._service)
self.__shutdownManifest(self._service, activeManifest)
except BaseException as excep:
self.__LOG.error('Cleanup failed - %s' % str(excep))
self._updateStatus(httpStatus=500,
error=errorCode,
errorMsg=errorMsg)
def test_post3_pkg_already_installed(self):
createManifest(self)
# now lets remove the manifest path
path = os.path.join(manifestutil.manifestPath('foo', 'bar'))
shutil.rmtree(path)
# now create the manifest again
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
def testDelete2(self):
createManifest(self)
# need to change the owner of the service dir to be cronus
servicePath = ServiceController.servicePath('foo')
uname = configutil.getAgentUser()
uid, gid = utils.getUidGid(uname)
rchown(servicePath, uid, gid)
response = self.app.delete(url(controller='service', service='foo', action='delete'))
assert response.status_int == 200, "HTTP response != 200"
body = json.loads(response.body)
tm = time.time()
while (tm + 5 > time.time()):
response = self.app.get(body['status'], expect_errors = True)
LOG.debug ('************* Status response body = ' + response.body)
body = json.loads(response.body)
if (body['progress'] == 100):
break
assert not os.path.isdir(ServiceController.servicePath('foo'))
def test_post3_already_installed_manifest(self):
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']})
response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertEquals(201, response.status_int, 'Manifest Post assert')
body = json.loads(response.body)
assert response.status_int == 201, 'Manifest Post assert'
def test_post3_already_installed_manifest(self):
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']})
response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertEquals(201, response.status_int, 'Manifest Post assert')
body = json.loads(response.body)
assert response.status_int == 201, 'Manifest Post assert'
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
try:
activePath = os.path.join(ServiceController.manifestPath(self._service), 'active')
oldManifest = None
# make sure that if the active path exists, it's a link
# if not log that and delete the link
if (os.path.exists(activePath) and not os.name == 'nt' and not islink(activePath)):
self.__LOG.error('%s is not a link. Attempted to delete' % activePath)
shutil.rmtree(activePath)
if (os.path.exists(activePath)):
oldManifest = os.path.basename(readlink(activePath))
else:
raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, errorMsg = 'No active manifest - cannot deactivate service')
self.__deactivateManifest(self._service, oldManifest)
self.__removeSymlink(self._service)
except SystemExit as exc:
failed = True
if (len(exc.args) == 2):
# ok we got {err code, err msg}
errorCode = exc.args[0]
errorMsg = exc.args[1]
raise exc
except AgentException as exc:
failed = True
errorMsg = 'Deactivate Manifest - Agent Exception - %s' % exc.getMsg()
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Deactivate Manifest - Unknown error - (%s) - %s - %s' \
% (self._service, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.warning(errorMsg)
self._updateStatus(httpStatus = 500, error = errorCode,
errorMsg = errorMsg)
self.__LOG.debug('Done: activate manifest for (%s)' % (self._service))
self._updateProgress(100)
def test_get_installed_pkgs_filter_inprogress(self):
from agent.lib.package import PackageUtil
serviceName = '.sbe.appService.SI1'
manifestName = 'manifestA'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf'
installed_pkg_path = ServiceController.installedPkgPath(serviceName)
installed_packages = PackageUtil.getAllInstalledPackages(installed_pkg_path)
self.assertEquals(2, len(installed_packages))
for path in installed_packages:
os.mkdir(path + '.inprogress')
installed_packages = PackageUtil.getAllInstalledPackages(installed_pkg_path)
self.assertEquals(2, len(installed_packages))
def delete(self, service, manifest):
""" Delete a new service object """
try:
path = ManifestController.manifestPath(service, manifest)
if (not os.path.isdir(path)):
return errorResult(request,
response,
Errors.MANIFEST_NOT_FOUND,
'manifest (%s/%s) missing service' %
(service, manifest),
controller=self)
# first check that this isn't the active manifest
path = os.path.join(ServiceController.manifestPath(service),
'active')
if (os.path.exists(path)):
activePath = os.path.basename(readlink(path))
deletePath = os.path.basename(
ManifestController.manifestPath(service, manifest))
if (activePath == deletePath):
return errorResult(
request,
response,
Errors.MANIFEST_DELETING_ACTIVE_MANIFEST,
'Manifest(%s, %s) attempting to delete active manifest'
% (service, manifest),
controller=self)
# now try to delete the manifest directory
appGlobal = config['pylons.app_globals']
manThread = ManifestDelete(appGlobal.threadMgr, service, manifest)
self.injectJobCtx(manThread)
manThread.start()
manThread.threadMgrEvent.wait()
return statusResult(request, response, manThread, controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error for delete manifest(%s/%s) - %s - %s' %
(service, manifest, str(excep), traceback.format_exc(2)),
controller=self)
def test_get_installed_pkgs_filter_inprogress(self):
from agent.lib.package import PackageUtil
serviceName = '.sbe.appService.SI1'
manifestName = 'manifestA'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf'
installed_pkg_path = ServiceController.installedPkgPath(serviceName)
installed_packages = PackageUtil.getAllInstalledPackages(installed_pkg_path)
self.assertEquals(2, len(installed_packages))
for path in installed_packages:
os.mkdir(path + '.inprogress')
installed_packages = PackageUtil.getAllInstalledPackages(installed_pkg_path)
self.assertEquals(2, len(installed_packages))
def testGet(self):
path = manifestutil.manifestPath('foo', 'bar')
os.makedirs(path)
path = manifestutil.manifestPath('foo', 'baz')
os.makedirs(path)
activePath = os.path.join(ServiceController.manifestPath('foo'), 'active')
symlink('bar', activePath)
response = self.app.get(url(controller='service', service='foo', action='get'), expect_errors = True)
body = json.loads(response.body)
print "************** response = %s" % body
assert body['progress'] == 100
assert body['result']['activemanifest'] == 'bar'
assert body['result']['manifest'] == ['bar', 'baz']
def test_package_reuse(self):
createManifest(self)
body = json.dumps({'package' : ['/packages/perlserver']})
body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
for _ in range(10):
if islink(os.path.join(manifestutil.manifestPath('foo', 'baz'), 'perlserver')):
break
time.sleep(1)
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'baz'), 'perlserver')))
def test_package_reuse(self):
createManifest(self)
body = json.dumps({'package' : ['/packages/perlserver']})
body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
for _ in range(10):
if islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')):
break
time.sleep(1)
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')))
def test_cleanupOrphanedPackages(self):
from agent.lib.utils import rchown
from agent.controllers.service import ServiceController
from agent.controllers.manifest import ManifestController
from agent.tests.unit.test_util import createManifest
if os.name == 'nt':
pass
else:
serviceName = 'service2'
manifestName = 'manifestA'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
installedPkgPath = os.path.realpath(
os.path.join(ServiceController.serviceRootPath(), serviceName,
'installed-packages'))
if (not os.name == 'nt'):
import pwd
uname = pylons.config['agent_user_account']
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
rchown(installedPkgPath, uid, gid)
#installedPkgPath = os.path.realpath(os.path.join('/ebay/srengarajan/e2e/local-deploy/service_nodes/' , '.sbe.appService.SI1', 'installed-packages' ))
#pkgs = os.listdir('C:\\srini\\projects\\gc\\agent\\trunk')
self.assertEquals(len(os.listdir(installedPkgPath)), 1)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
pylons.config['packageMgr_install_package_age'] = 0
pylons.config['packageMgr_install_package_min_age'] = 0.0
time.sleep(1)
PackageUtil.cleanupOrphanedPackages()
self.assertEquals(len(os.listdir(installedPkgPath)), 0)
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
activeManifest = None
try:
activePath = os.path.join(ServiceController.manifestPath(self._service), 'active')
# make sure that the active path exists and it is a link
# Should we check this again since we already have a check in action controller
if not os.path.exists(activePath) or not islink(activePath):
raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, errorMsg = 'No active manifest - cannot reset service')
activeManifest = os.path.basename(readlink(activePath))
self.__shutdownManifest(self._service, activeManifest)
self.__deactivateManifest(self._service, activeManifest)
self.__activateManifest(self._service, activeManifest)
self.__startupManifest(self._service, activeManifest)
self.__LOG.info('Done: reset service for (%s/%s)' % (self._service, activeManifest))
self._updateStatus(progress = 100)
except AgentException as exc:
failed = True
errorMsg = 'Activate Manifest - Agent Exception - %s' % exc.getMsg()
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Activate Manifest - Unknown error - (%s/%s) - %s - %s' \
% (self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.error(errorMsg)
if self._service and activeManifest:
try:
self.__LOG.info('Reset service %s failed, shutdown to cleanup' % self._service)
self.__shutdownManifest(self._service, activeManifest)
except BaseException as excep:
self.__LOG.error('Cleanup failed - %s' % str(excep))
self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
def test_post3_pkg_already_installed(self):
createManifest(self)
# now lets remove the manifest path
path = os.path.join(ManifestController.manifestPath('foo', 'bar'))
if os.name == 'nt':
cmd = 'rm -r %s' % path
LOG.debug("running command %s" % cmd)
os.system(cmd)
else:
shutil.rmtree(path)
# now create the manifest again
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def testGet(self):
path = ManifestController.manifestPath('foo', 'bar')
os.makedirs(path)
path = ManifestController.manifestPath('foo', 'baz')
os.makedirs(path)
activePath = os.path.join(ServiceController.manifestPath('foo'),
'active')
symlink('bar', activePath)
response = self.app.get(url(controller='service',
service='foo',
action='get'),
expect_errors=True)
body = json.loads(response.body)
print "************** response = %s" % body
assert body['progress'] == 100
assert body['result']['activemanifest'] == 'bar'
assert body['result']['manifest'] == ['bar', 'baz']
def test_getAllInstalledPackages(self):
from agent.tests.unit.test_util import createManifest
from agent.controllers.service import ServiceController
serviceName = 'service1'
manifestName = 'manifestA'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
time.sleep(2)
installedPkgPath = os.path.join(
ServiceController.servicePath(serviceName), 'installed-packages')
pkgs = PackageUtil.getAllInstalledPackages(installedPkgPath)
self.assertEquals(pkgs.__len__(), 2)
def getAllSymLinks(service):
""" return all the symlinks from manifests to packages for a given service"""
LOG.debug('calling getAllSymLinks %s' % service)
linkedPaths = []
manPath = ServiceController.manifestPath(service)
LOG.debug('manifestPath is %s' % manPath)
for path in os.listdir(manPath):
LOG.debug('path is %s' % path)
if not islink(os.path.join(manPath, path)):
LOG.debug('path is dir not a link')
pkgPaths = [ packageDir for packageDir in os.listdir(os.path.join(manPath, path)) ]
LOG.debug('pkgPaths is %s' % pkgPaths)
for pkgPath in pkgPaths:
try:
LOG.debug('pkgPath is %s' % pkgPath)
if not os.path.isfile(os.path.join(manPath, path, pkgPath)) and islink(os.path.join(manPath, path, pkgPath)):
LOG.debug('pkgPaths is %s' % pkgPath)
targetPath = os.path.abspath(readlink(os.path.join(manPath, path, pkgPath)))
linkedPaths.append(targetPath)
LOG.debug('targetPath is %s ' % targetPath)
except BaseException as exc:
LOG.error('failed to read link for the pkg path %s' % str(exc))
return linkedPaths
def test_same_pkg_download_parallel(self):
packages = ['http://www.stackscaling.com/agentrepo/pkgA-1.2.0.unix.cronus']
manifest1 = 'bar'
manifest2 = 'blah'
service1 = 'foo'
service2 = 'lah'
try:
for pkg in packages:
mockDownloadPkg(pkg)
path = ServiceController.manifestPath(service1)
os.makedirs(path)
path = ServiceController.installedPkgPath(service1)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service1)
os.makedirs(path)
path = ServiceController.manifestPath(service2)
os.makedirs(path)
path = ServiceController.installedPkgPath(service2)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service2)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response1.status_int == 200, 'Manifest Post assert'
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response2.status_int == 200, 'Manifest Post assert'
checkStatus(self, 'create manifest bar', response1, timeout = 25)
checkStatus(self, 'create manifest baz', response2, timeout = 25)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'pkgA')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('lah', 'blah'), 'pkgA')))
def test_same_pkg_download_parallel(self):
packages = ['http://repository.qa.ebay.com/cronus/test-data/agent/perlserver-1.0.0.unix.cronus']
manifest1 = 'bar'
manifest2 = 'blah'
service1 = 'foo'
service2 = 'lah'
try:
path = ServiceController.manifestPath(service1)
os.makedirs(path)
path = ServiceController.installedPkgPath(service1)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service1)
os.makedirs(path)
path = ServiceController.manifestPath(service2)
os.makedirs(path)
path = ServiceController.installedPkgPath(service2)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service2)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response1.status_int == 200, 'Manifest Post assert'
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response2.status_int == 200, 'Manifest Post assert'
checkStatus(self, 'create manifest bar', response1, timeout = 25)
checkStatus(self, 'create manifest baz', response2, timeout = 25)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('lah', 'blah'), 'perlserver')))
def startAgentGlobals(startThreadMgr = True, startPackageMgr = True, startAgentMonitor = True):
"""
Create and start the global objects
"""
# cleanup incomplete manifests
from agent.controllers.service import ServiceController
ServiceController.cleanupServices()
# stop the existing agents
stopAgentGlobals()
appGlobal = pylons.config['pylons.app_globals']
# load config override from agent .metadata.json
configutil.loadPylonConfig(pylons.config)
configutil.loadConfigOverrides()
configutil.loadSecureConfigOverrides()
LOG.info("Complete loading config overrides")
# build in memory security token cache
appGlobal.authztoken = str(uuid.uuid4())
appGlobal.encryptedtokens = {}
agentauth.buildTokenCache(appGlobal.authztoken)
LOG.info("Complete building security token cache")
# agent health
appGlobal.agentHealth = 'True'
appGlobal.agentHealthFactor = None
from datetime import datetime
appGlobal.agentInfo = {'version': agenthealth.loadVersion(),
'startup': str(datetime.now())}
appGlobal.osInfo = agenthealth.getOsInfo()
appGlobal.diskOk = True
LOG.info("Agent health initialized")
# start thread mgr
appGlobal.threadMgr = ThreadMgr(garbageFreq = pylons.config['threadMgr_garbage_freq'],
maxThreadAge = pylons.config['threadMgr_thread_age'])
if startThreadMgr:
appGlobal.threadMgr.start()
LOG.info("Agent thread manager started")
# start package mgr
appGlobal.packageMgr = PackageMgr(garbageFreq = pylons.config['packageMgr_garbage_freq'],
maxPackageAge = pylons.config['packageMgr_package_age'])
if startPackageMgr:
appGlobal.packageMgr.start()
LOG.info("Agent package manager started")
#start monitor manager
from agent.lib.monitors.monitor import AgentMonitor
appGlobal.agentMonitor = AgentMonitor()
if startAgentMonitor:
appGlobal.agentMonitor.start()
LOG.info("Agent monitor started")
# Declare dictionary for storing dynamic controllers
appGlobal.dynacontrollers = dict()
# metrix manager initialization
from agent.lib.monitors.metrix_manager import MetrixManager
from agent.lib.monitors.system_monitor import SystemMonitor
appGlobal.metrixManager = MetrixManager()
appGlobal.systemMonitor = SystemMonitor()
appGlobal.metrixManager.register('Configuration', validate_internals.getConfigFileFiltered, 6)
appGlobal.metrixManager.register('Configuration Overrides', configutil.getConfigOverrides, 7)
appGlobal.metrixManager.register(AGENT_HEALTH_VI_KEY, lambda : appGlobal.agentHealth, 1)
appGlobal.metrixManager.register(AGENT_HEALTH_FACTOR_VI_KEY, lambda : appGlobal.agentHealthFactor, 1)
appGlobal.metrixManager.register(OS_INFO_KEY, lambda : appGlobal.osInfo, 3)
appGlobal.metrixManager.register(AGENT_INFO_KEY, lambda : appGlobal.agentInfo, 2)
LOG.info("Agent health metrics registered")
# start all agent modules
modulestartthread = threading.Thread(target = ModuleController.loadModuleOnAgentStartup)
modulestartthread.start()
LOG.info("Local modules started")
# start all services with active manifest, and load dynamic controllers
servicestartthread = threading.Thread(target = ServiceController.startServicesOnAgentStartup)
servicestartthread.start()
LOG.info("Local services started")
appGlobal.sdutil = shutdownAgent
