def test_multiple_manifest_create(self):
""" when a manifest creation is in progress for a service, another creation should block """
packages = ["http://repository.qa.ebay.com/cronus/test-data/agent/pkgA-1.2.0.unix.cronus"]
service = 'foo'
manifest1 = 'bar'
manifest2 = 'car'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
self.assertEquals(response1.status_int, 200, 'Manifest1 Post assert - should go through')
try:
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
self.assertFalse(True, 'Expected an exception but did not get one!')
except AppError:
pass
checkStatus(self, 'create manifest bar', response1, timeout = 25)
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'pkgA')))
self.assertFalse(islink(os.path.join(ManifestController.manifestPath('foo', 'car'), 'pkgA')))
def test_post_manifest_inprogress_ok(self):
service = 'foo'
manifest = 'blahblah'
try:
path = ServiceController.servicePath(service)
if os.path.exists(path):
if os.name == 'nt':
cmd = 'rm -r %s' % path
LOG.debug("running command %s" % cmd)
os.system(cmd)
else:
shutil.rmtree(path)
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(service, manifest))
os.makedirs(inProgressPath)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus"], manifest = 'blahblah', createDirs = False)
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'blahblah'), 'pkgA')))
def __delAllExceptActiveManifests(self, service):
'''
Delete all manifests in the given service except active link and corresponding manifest (if one exists).
Doesn't delete the corresponding dir in installed-packages (similar to 'delete manifest')
'''
manifests = ManifestController.getManifests(service)
active = ManifestController.getActiveManifest(service)
if active and len(active) != 0: # 'active' can be none
manifests.remove(active)
for mft in manifests:
mf_path = ManifestController.manifestPath(service, mft)
ServiceDelete.deleteFolderContents(mf_path)
def __delAllExceptActiveManifests(self, service):
'''
Delete all manifests in the given service except active link and corresponding manifest (if one exists).
Doesn't delete the corresponding dir in installed-packages (similar to 'delete manifest')
'''
manifests = ManifestController.getManifests(service)
active = ManifestController.getActiveManifest(service)
if active and len(active) != 0: # 'active' can be none
manifests.remove(active)
for mft in manifests:
mf_path = ManifestController.manifestPath(service, mft)
ServiceDelete.deleteFolderContents(mf_path)
def test_delete(self):
createManifest(self)
response = self.app.delete(url(controller = 'manifest', action = 'delete', service = "foo", manifest = "bar"),
expect_errors = True)
print '*************** = ' + response.body
self.assertEquals(200, response.status_int)
for _ in range(10):
if not os.path.isdir(ManifestController.manifestPath('foo', 'bar')):
break
else:
time.sleep(0.1)
self.assertFalse(os.path.isdir(ManifestController.manifestPath('foo', 'bar')))
def test_getAllSymLinks(self):
#import pdb;
#pdb.set_trace()
serviceName = '.sbe.appService.SI1'
manifestName = 'manifestA'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf'
symLinks = ManifestController.getAllSymLinks(serviceName)
""" return all the symlinks from manifests to packages for a given service"""
LOG.debug('calling getAllSymLinks')
#serviceName = '.sbe.appService.SI1'
linkedPaths = []
#pdb.set_trace()
#manPath = '/ebay/srengarajan/e2e/local-deploy/service_nodes/.sbe.appService.SI1/manifests'
installedPkgPath = os.path.join(ServiceController.servicePath(serviceName), 'installed-packages')
pathToPkgA = os.path.join(installedPkgPath, 'pkgA')
pathToPkgA120 = os.path.join(pathToPkgA, '1.2.0.unix')
pathToPkgB = os.path.join(installedPkgPath, 'pkgB')
pathToPkgB060 = os.path.join(pathToPkgB, '0.6.0.unix')
bitmap = 0
for path in symLinks:
if path.find(pathToPkgA120) >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 1
elif path.find(pathToPkgB060) >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 2
self.assertEquals(3, bitmap)
def reset(self, service):
''' Controller to reset service '''
LOG.info('reset for service(%s)', service)
try:
appGlobal = config['pylons.app_globals']
if not os.path.exists(
ManifestController.manifestPath(service, 'active')):
return errorResult(request,
response,
Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Active Manifest(%s) path missing' %
(service),
controller=self)
resetThread = ResetService(appGlobal.threadMgr, service)
self.injectJobCtx(resetThread)
resetThread.start()
resetThread.threadMgrEvent.wait()
return statusResult(request,
response,
resetThread,
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error for restart service(%s) - %s - %s' %
(service, str(excep), traceback.format_exc(2)),
controller=self)
def cleanupOrphanedPackages(checkAge = False):
''' API to cleanup Orphaned Packages For All Services '''
from agent.controllers.service import ServiceController
#services = os.listdir(service_nodes)
from agent.controllers.manifest import ManifestController
#import pdb;pdb.set_trace()
services = ServiceController.getServices()
service_nodes = ServiceController.serviceRootPath()
linkedPaths = []
sysServices = ['agent']
LOG.info('Garbage collecting orphaned installed packages')
for service in services:
try:
if service in sysServices:
LOG.debug('system services cannot be garbage collected')
else:
servicePath = os.path.join(service_nodes, service)
installedPkgPath = os.path.join(servicePath, 'installed-packages')
linkedPaths.extend(ManifestController.getAllSymLinks(service))
linkedPaths.extend(manifestutil.getModuleSymLinks(service))
LOG.debug('symLinks returned %s' % linkedPaths)
installedPkgPaths = PackageUtil.getAllInstalledPackages(installedPkgPath)
LOG.debug('installedPkgPaths returned for the service %s' % installedPkgPaths)
if len(installedPkgPaths) > 0:
orphanPkgs = set(installedPkgPaths) - set(linkedPaths)
LOG.debug('orphanPkgs returned %s' % orphanPkgs)
PackageUtil.cleanupInstalledPkgs(installedPkgPath, orphanPkgs)
except BaseException as excep:
LOG.error('Failed to proceed with garbage collection %s' % str(excep))
# agent-804, manifests only contains folders, need to delete if file is in manifests
servicePath = os.path.join(service_nodes, service)
if not os.path.isdir(servicePath):
utils.runsyscmd('rm -f %s' % servicePath)
LOG.info('Garbage collecting orphaned installed packages completed')
def activatemanifest(self, service):
""" activate manifest, if already active then skip """
LOG.info('activateManifest for service(%s) with body: %s', service, request.body)
try:
appGlobal = config['pylons.app_globals']
manifest = ''
requestjson = json.loads(request.body)
manifest = requestjson['manifest']
force = asbool(requestjson['force']) if 'force' in requestjson else False
if not force and manifestutil.getActiveManifest(service) == manifest:
return doneResult(request, response, controller=self)
else:
mf_path = os.path.join(ManifestController.manifestPath(service, manifest))
if (not os.path.exists(mf_path)):
return errorResult(request, response, Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Manifest(%s, %s) path missing' % (service, manifest),
controller = self)
LOG.debug('Manifest path exists: %s' % (mf_path))
activateThread = ActivateManifest(appGlobal.threadMgr, service, manifest)
self.injectJobCtx(activateThread)
activateThread.start()
activateThread.threadMgrEvent.wait()
return statusResult(request, response, activateThread, controller = self)
except Exception as excep:
msg = 'Unknown error for activateManifest(%s/%s) - %s - %s' % (service, manifest, str(excep), traceback.format_exc(2))
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = msg, controller = self)
def test_post2(self):
# successful post
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_package_reuse(self):
createManifest(self)
body = json.dumps({'package' : ['/packages/perlserver']})
body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
for _ in range(10):
if islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')):
break
time.sleep(1)
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')))
def _obtainCmdExecThreads(self, exeName, service, manifest, activateFlow):
"""start the threads to execute cmd in each package in the service/manifest"""
execThreads = []
packages = ManifestController.getPackages(service,
os.path.basename(manifest))
# Executing the deactivate in reversed order where the last package is first deactivated.
if (not activateFlow):
packages.reverse()
# make sure there's something to do, otherwise just return here
if (len(packages) == 0):
return execThreads
for package in packages:
pkgPath = manifestutil.packagePath(service, manifest, package)
pkgInit = PkgInitConfig(pkgPath)
pkgConfig = pkgInit.getConfigs()
# check command type and package startup process
if (exeName == 'reboot' or exeName == 'restart'):
# default action for reboot is to startup, noop for restart agent
action = 'startup' if 'reboot' == exeName else 'noop'
if exeName == 'reboot' and pkgConfig and pkgConfig.has_key(
'lcm_on_system_reboot'):
action = pkgConfig['lcm_on_system_reboot']
if (exeName == 'restart' and pkgConfig
and pkgConfig.has_key('lcm_on_agent_restart')):
action = pkgConfig['lcm_on_agent_restart']
if action == 'reset':
execThread, dummy = self._getBuiltThread(
service, manifest, package, "shutdown")
execThreads.append([execThread, dummy])
execThread, dummy = self._getBuiltThread(
service, manifest, package, "deactivate")
execThreads.append([execThread, dummy])
execThread, dummy = self._getBuiltThread(
service, manifest, package, "activate")
execThreads.append([execThread, dummy])
if (action == 'reset' or action == 'startup'):
execThread, dummy = self._getBuiltThread(
service, manifest, package, 'startup')
execThreads.append([execThread, dummy])
else:
execThread, dummy = self._getBuiltThread(
service, manifest, package, exeName)
execThreads.append([execThread, dummy])
return execThreads
def test_post_without_service(self):
body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']})
response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response.status_int == 200, 'Manifest Post assert'
time.sleep(1)
self.assertFalse(os.path.exists(ManifestController.manifestPath('foo', 'bar')))
def test_post3_pkg_already_installed(self):
createManifest(self)
# now lets remove the manifest path
path = os.path.join(ManifestController.manifestPath('foo', 'bar'))
if os.name == 'nt':
cmd = 'rm -r %s' % path
LOG.debug("running command %s" % cmd)
os.system(cmd)
else:
shutil.rmtree(path)
# now create the manifest again
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def testGet(self):
path = ManifestController.manifestPath('foo', 'bar')
os.makedirs(path)
path = ManifestController.manifestPath('foo', 'baz')
os.makedirs(path)
activePath = os.path.join(ServiceController.manifestPath('foo'),
'active')
symlink('bar', activePath)
response = self.app.get(url(controller='service',
service='foo',
action='get'),
expect_errors=True)
body = json.loads(response.body)
print "************** response = %s" % body
assert body['progress'] == 100
assert body['result']['activemanifest'] == 'bar'
assert body['result']['manifest'] == ['bar', 'baz']
def test_same_pkg_download_parallel(self):
packages = ['http://repository.qa.ebay.com/cronus/test-data/agent/perlserver-1.0.0.unix.cronus']
manifest1 = 'bar'
manifest2 = 'blah'
service1 = 'foo'
service2 = 'lah'
try:
path = ServiceController.manifestPath(service1)
os.makedirs(path)
path = ServiceController.installedPkgPath(service1)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service1)
os.makedirs(path)
path = ServiceController.manifestPath(service2)
os.makedirs(path)
path = ServiceController.installedPkgPath(service2)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service2)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response1.status_int == 200, 'Manifest Post assert'
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response2.status_int == 200, 'Manifest Post assert'
checkStatus(self, 'create manifest bar', response1, timeout = 25)
checkStatus(self, 'create manifest baz', response2, timeout = 25)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('lah', 'blah'), 'perlserver')))
def _getBuiltThread(self, service, manifest, package, exeName):
""" here """
# figure out the path to the cronus scripts
uname = pylons.config['app_user_account']
execPath = os.path.join(ManifestController.manifestPath(service, manifest), package, 'cronus', 'scripts', exeName)
if (isHigherPrivilegeService(service)):
cmd = execPath
else:
cmd = utils.sudoCmd([execPath], uname)
dummy = not os.path.exists(execPath)
execThread = ExecThread(self._threadMgr, cmd)
copycontexts(self, execThread, contextutils.CTX_NAMES)
return execThread, dummy
def _obtainCmdExecThreads(self, exeName, service, manifest, activateFlow):
"""start the threads to execute cmd in each package in the service/manifest"""
execThreads = []
packages = ManifestController.getPackages(service, os.path.basename(manifest))
# Executing the deactivate in reversed order where the last package is first deactivated.
if (not activateFlow):
packages.reverse()
# make sure there's something to do, otherwise just return here
if (len(packages) == 0):
return execThreads
for package in packages:
pkgPath = manifestutil.packagePath(service, manifest, package)
pkgInit = PkgInitConfig(pkgPath)
pkgConfig = pkgInit.getConfigs()
# check command type and package startup process
if (exeName == 'reboot' or exeName == 'restart'):
# default action for reboot is to startup, noop for restart agent
action = 'startup' if 'reboot' == exeName else 'noop'
if exeName == 'reboot' and pkgConfig and pkgConfig.has_key('lcm_on_system_reboot'):
action = pkgConfig['lcm_on_system_reboot']
if (exeName == 'restart' and pkgConfig and pkgConfig.has_key('lcm_on_agent_restart')):
action = pkgConfig['lcm_on_agent_restart']
if action == 'reset':
execThread, dummy = self._getBuiltThread(service, manifest, package, "shutdown")
execThreads.append([execThread, dummy])
execThread, dummy = self._getBuiltThread(service, manifest, package, "deactivate")
execThreads.append([execThread, dummy])
execThread, dummy = self._getBuiltThread(service, manifest, package, "activate")
execThreads.append([execThread, dummy])
if (action == 'reset' or action == 'startup'):
execThread, dummy = self._getBuiltThread(service, manifest, package, 'startup')
execThreads.append([execThread, dummy])
else:
execThread, dummy = self._getBuiltThread(service, manifest, package, exeName)
execThreads.append([execThread, dummy])
return execThreads
def _getBuiltThread(self, service, manifest, package, exeName):
""" here """
# figure out the path to the cronus scripts
uname = pylons.config['app_user_account']
execPath = os.path.join(
ManifestController.manifestPath(service, manifest), package,
'cronus', 'scripts', exeName)
if (isHigherPrivilegeService(service)):
cmd = execPath
else:
cmd = utils.sudoCmd([execPath], uname)
dummy = not os.path.exists(execPath)
execThread = ExecThread(self._threadMgr, cmd)
copycontexts(self, execThread, contextutils.CTX_NAMES)
return execThread, dummy
def test_post3_already_installed_manifest(self):
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']})
response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertEquals(201, response.status_int, 'Manifest Post assert')
body = json.loads(response.body)
assert response.status_int == 201, 'Manifest Post assert'
def activatemanifest(self, service):
""" activate manifest, if already active then skip """
LOG.info('activateManifest for service(%s) with body: %s', service,
request.body)
try:
appGlobal = config['pylons.app_globals']
manifest = ''
requestjson = json.loads(request.body)
manifest = requestjson['manifest']
force = asbool(
requestjson['force']) if 'force' in requestjson else False
if not force and manifestutil.getActiveManifest(
service) == manifest:
return doneResult(request, response, controller=self)
else:
mf_path = os.path.join(
ManifestController.manifestPath(service, manifest))
if (not os.path.exists(mf_path)):
return errorResult(request,
response,
Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Manifest(%s, %s) path missing' %
(service, manifest),
controller=self)
LOG.debug('Manifest path exists: %s' % (mf_path))
activateThread = ActivateManifest(appGlobal.threadMgr, service,
manifest)
self.injectJobCtx(activateThread)
activateThread.start()
activateThread.threadMgrEvent.wait()
return statusResult(request,
response,
activateThread,
controller=self)
except Exception as excep:
msg = 'Unknown error for activateManifest(%s/%s) - %s - %s' % (
service, manifest, str(excep), traceback.format_exc(2))
return errorResult(request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg=msg,
controller=self)
def test_inprogress_pkg_download(self):
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress')
inprogressFile = open(inprogressPath, 'w')
inprogressFile.write('somegarbage')
inprogressFile.close()
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def cleanupOrphanedPackages(checkAge=False):
''' API to cleanup Orphaned Packages For All Services '''
from agent.controllers.service import ServiceController
#services = os.listdir(service_nodes)
from agent.controllers.manifest import ManifestController
#import pdb;pdb.set_trace()
services = ServiceController.getServices()
service_nodes = ServiceController.serviceRootPath()
linkedPaths = []
sysServices = ['agent']
LOG.info('Garbage collecting orphaned installed packages')
for service in services:
try:
if service in sysServices:
LOG.debug('system services cannot be garbage collected')
else:
servicePath = os.path.join(service_nodes, service)
installedPkgPath = os.path.join(servicePath,
'installed-packages')
linkedPaths.extend(
ManifestController.getAllSymLinks(service))
linkedPaths.extend(manifestutil.getModuleSymLinks(service))
LOG.debug('symLinks returned %s' % linkedPaths)
installedPkgPaths = PackageUtil.getAllInstalledPackages(
installedPkgPath)
LOG.debug('installedPkgPaths returned for the service %s' %
installedPkgPaths)
if len(installedPkgPaths) > 0:
orphanPkgs = set(installedPkgPaths) - set(linkedPaths)
LOG.debug('orphanPkgs returned %s' % orphanPkgs)
PackageUtil.cleanupInstalledPkgs(
installedPkgPath, orphanPkgs)
except BaseException as excep:
LOG.error('Failed to proceed with garbage collection %s' %
str(excep))
# agent-804, manifests only contains folders, need to delete if file is in manifests
servicePath = os.path.join(service_nodes, service)
if not os.path.isdir(servicePath):
utils.runsyscmd('rm -f %s' % servicePath)
LOG.info('Garbage collecting orphaned installed packages completed')
def test_getPackagePaths(self):
serviceName = 'service1'
manifestName = 'manifestA'
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
packagePaths = ManifestController.getPackagePaths(serviceName, manifestName);
print ">>>>>>>>>>>>" + str(packagePaths)
self.assertTrue(len(packagePaths) >= 2)
bitmap = 0
for path in packagePaths:
if path.find("pkgA") >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 1
elif path.find("pkgB") >= 0:
self.assertTrue(os.path.isdir(path))
bitmap |= 2
self.assertEquals(3, bitmap)
def reset(self, service):
''' Controller to reset service '''
LOG.info('reset for service(%s)', service)
try:
appGlobal = config['pylons.app_globals']
if not os.path.exists(ManifestController.manifestPath(service, 'active')):
return errorResult(request, response, Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Active Manifest(%s) path missing' % (service),
controller = self)
resetThread = ResetService(appGlobal.threadMgr, service)
self.injectJobCtx(resetThread)
resetThread.start()
resetThread.threadMgrEvent.wait()
return statusResult(request, response, resetThread, controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error for restart service(%s) - %s - %s' %
(service, str(excep), traceback.format_exc(2)),
controller = self)
def testDelete(self):
def deleteTestDir(service):
LOG.debug('************ service = %s' % service)
response = self.app.delete(url(controller='service', service=service, action='delete'), expect_errors = True)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
# assert response.status_int == 500, "HTTP response != 500"
def makeTestDir(path):
os.makedirs(path)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'installed-packages'))
def createManifests(mf_path):
os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
latest = os.path.join(mf_path, 'm3.0')
os.makedirs(os.path.join(latest, 'dummy_dir3.0'))
utils.symlink(latest, os.path.join(mf_path, 'active'))
return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')
def makePackageContent(path, pkgPath, pkgPropPath):
pkgFile = file(pkgPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
pkgFile = file(pkgPropPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(path, uname)
def createTestThread(serviceName):
appGlobal = config['pylons.app_globals']
testTh = WaitThread(appGlobal.threadMgr, ServiceController.serviceCat(serviceName))
testTh.start()
return testTh
def startTestProcess():
cmd = utils.sudoCmd(["sleep", "5"], pylons.config['app_user_account'])
return Popen(cmd)
if os.name == 'nt':
LOG.warning('Services cleanup not supported on windows. Skipping test...')
return
path1 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'foo')
path2 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'bar')
path3 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'agent')
deleteTestDir('foo')
deleteTestDir('bar')
deleteTestDir('agent')
# make dirs
makeTestDir(path1)
makeTestDir(path2)
makeTestDir(path3)
all_mf, active_mf = createManifests(ServiceController.manifestPath('agent'))
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(ServiceController.serviceRootPath(), uname)
pkgDir = PackageMgr.packagePath()
pkgPath = os.path.join(pkgDir, "foo.cronus")
pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop")
makePackageContent(pkgDir, pkgPath, pkgPropPath)
# create threads
testThFoo = createTestThread('foo')
testThBar = createTestThread('bar')
testThAgent = createTestThread('agent')
# start process
process = startTestProcess()
# start testing
LOG.debug('************ start cleanup')
response = self.app.post(url(controller='cleanup', action='post'))
LOG.debug ('Delete response body = ' + response.body)
body = json.loads(response.body)
tm = time.time()
while (tm + 10 > time.time()):
response = self.app.get(body['status'], expect_errors = True)
LOG.debug ('Status response body = ' + response.body)
body = json.loads(response.body)
print body
if (body['progress'] == 100):
break
time.sleep(0.1)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
time.sleep(0.1)
assert not os.path.exists(path1), 'service foo does exist or is not a directory'
assert not os.path.exists(path2), 'service bar does exist or is not a directory'
assert os.path.exists(path3), 'service agent does NOT exist or is not a directory'
assert not testThFoo.isAlive(), 'thread Foo is still alive'
assert not testThBar.isAlive(), 'thread Bar is still alive'
assert not testThAgent.isAlive(), 'thread Agent is still alive'
assert not os.path.exists(pkgPath), 'package foo exists'
assert not os.path.exists(pkgPropPath), 'package prop foo exists'
assert os.path.exists(pkgDir), 'package directory does not exist'
# ensure agent cleanup is proper
active_mf_path = ManifestController.manifestPath('agent', active_mf)
active_link = os.path.join(ServiceController.manifestPath('agent'), 'active')
all_mf.remove(active_mf)
actual_active_mf_path = utils.readlink(active_link)
self.assertTrue(os.path.exists(active_mf_path), 'active agent manifest got deleted but shouldn\t have')
self.assertTrue(os.path.exists(active_link), 'agent active link missing')
self.assertEqual(active_mf_path, actual_active_mf_path, 'agent active link pointing to some wrong manifest; link broken?')
for mf in all_mf:
agnt_mf_path = ManifestController.manifestPath('agent', mf)
self.assertFalse(os.path.exists(agnt_mf_path), 'non active agent mf %s should have been deleted' % mf)
def doRun(self):
""" Main body of the thread
Progress Info:
Look at the list of packages this manifest has and figure out the progress of this manifest
Progress 1 = manifest directory created
Progress 2-80 = all packages downloaded
Progress 81-99 = all packages untarred
Progress 100 = all links done
"""
inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(self.__service, self.__manifest))
try:
if self.__service != 'agent':
utils.checkDiskFull()
installedPkgPath = ServiceController.installedPkgPath(self.__service)
# This shouldn'trn happen but make sure there isn'trn already a manifest directory in progress
if (os.path.isdir(inProgressPath)):
# probably another manifest create thread died out half way.
# Cleanup and reattempt manifest creation
LOG.debug('Manifest inprogress found for service/manifest (%s/%s). Will cleanup and retry' % (self.__service, self.__manifest))
shutil.rmtree(inProgressPath)
# make sure that the service path and installed package path exists
if (not os.path.isdir(ServiceController.manifestPath(self.__service)) or
not os.path.isdir(ServiceController.installedPkgPath(self.__service))):
errCode = Errors.SERVICE_NOT_FOUND
msg = 'Service (%s) for manifest (%s) not found' % (self.__service, self.__manifest)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
return
# ok create the manifest path
os.mkdir(inProgressPath)
self._updateProgress(1)
# figure out which of the packages are already there
remainingPackages = {}
for pkgUri in self.__packages:
pkgDict = PackageUtil.parseUri(pkgUri)
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
pkgName = pkgDict['packageName']
if (not os.path.exists(pkgPath)) or ((self.__forcePackages is not None) and pkgName in self.__forcePackages):
remainingPackages[pkgUri] = pkgDict
else:
symlink(pkgPath, os.path.join(inProgressPath, pkgDict['packageName']))
if self.__attemptDownload:
# now make sure all the packages are downloaded
try:
self._downloadPackages(remainingPackages.keys(), skipProp = self.__skipProp)
except AgentException as exc:
# check if it is download error, then modify exception appropriately
if exc.getCode() == Errors.DC_FAILED_DOWNLOAD:
exc = AgentException(Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED,
'Manifest (%s/%s) failed downloading package - %s'
% (self.__service, self.__manifest, exc.getMsg()))
raise exc
else:
if len(remainingPackages) > 0:
raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
'Create Manifest (%s/%s) failed since package is not present and download has been disabled'
% (self.__service, self.__manifest))
LOG.info('Completed download all packages for (%s/%s)' % (self.__service, self.__manifest))
# now untar the packages
import re
pkgSuffix = '.%s' % re.sub(r"\W", "", self.__manifest)
self._untarPackages(remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service), 0, self.__forcePackages, pkgSuffix)
LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__manifest))
# now create the links
for pkgDict in remainingPackages.itervalues():
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
linkPath = os.path.join(inProgressPath, pkgDict['packageName'])
# validate target folder does exist
if not os.path.exists(pkgPath):
raise AgentException(Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath)
symlink(pkgPath, linkPath)
# now move the inProgressPath to the final path
manifestContentPath = ManifestController.manifestContentPath(self.__service, self.__manifest)
os.rename(inProgressPath, ManifestController.manifestPath(self.__service, self.__manifest))
mfContentsFile = file(manifestContentPath, 'w')
for pkgUri in self.__packages:
mfContentsFile.write(('%s%s') % (pkgUri, os.linesep))
mfContentsFile.close()
LOG.info('Completed create manifest directories for (%s/%s)' % (self.__service, self.__manifest))
LOG.info('Completed create manifest for (%s/%s)' % (self.__service, self.__manifest))
self._updateStatus(progress = 100)
except AgentException as exc:
LOG.info(exc.getMsg())
self._updateStatus(httpStatus = 500, error = exc.getCode(), errorMsg = exc.getMsg())
except Exception as exc:
errCode = Errors.UNKNOWN_ERROR
msg = 'Unknown error for (%s/%s) - %s - %s' % (self.__service, self.__manifest,
str(exc), traceback.format_exc(2))
LOG.info(msg)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
finally:
# clean up intermediate progress
try:
shutil.rmtree(inProgressPath)
except OSError:
pass
def testDelete(self):
def deleteTestDir(service):
LOG.debug('************ service = %s' % service)
response = self.app.delete(url(controller='service',
service=service,
action='delete'),
expect_errors=True)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
# assert response.status_int == 500, "HTTP response != 500"
def makeTestDir(path):
os.makedirs(path)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'installed-packages'))
def createManifests(mf_path):
os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
latest = os.path.join(mf_path, 'm3.0')
os.makedirs(os.path.join(latest, 'dummy_dir3.0'))
utils.symlink(latest, os.path.join(mf_path, 'active'))
return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')
def makePackageContent(path, pkgPath, pkgPropPath):
pkgFile = file(pkgPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
pkgFile = file(pkgPropPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(path, uname)
def createTestThread(serviceName):
appGlobal = config['pylons.app_globals']
testTh = WaitThread(appGlobal.threadMgr,
ServiceController.serviceCat(serviceName))
testTh.start()
return testTh
def startTestProcess():
cmd = utils.sudoCmd(["sleep", "5"],
pylons.config['app_user_account'])
return Popen(cmd)
if os.name == 'nt':
LOG.warning(
'Services cleanup not supported on windows. Skipping test...')
return
path1 = os.path.join(pylons.config['agent_root'], 'service_nodes',
'foo')
path2 = os.path.join(pylons.config['agent_root'], 'service_nodes',
'bar')
path3 = os.path.join(pylons.config['agent_root'], 'service_nodes',
'agent')
deleteTestDir('foo')
deleteTestDir('bar')
deleteTestDir('agent')
# make dirs
makeTestDir(path1)
makeTestDir(path2)
makeTestDir(path3)
all_mf, active_mf = createManifests(
ServiceController.manifestPath('agent'))
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(ServiceController.serviceRootPath(),
uname)
pkgDir = PackageMgr.packagePath()
pkgPath = os.path.join(pkgDir, "foo.cronus")
pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop")
makePackageContent(pkgDir, pkgPath, pkgPropPath)
# create threads
testThFoo = createTestThread('foo')
testThBar = createTestThread('bar')
testThAgent = createTestThread('agent')
# start process
process = startTestProcess()
# start testing
LOG.debug('************ start cleanup')
response = self.app.post(url(controller='cleanup', action='post'))
LOG.debug('Delete response body = ' + response.body)
body = json.loads(response.body)
tm = time.time()
while (tm + 10 > time.time()):
response = self.app.get(body['status'], expect_errors=True)
LOG.debug('Status response body = ' + response.body)
body = json.loads(response.body)
print body
if (body['progress'] == 100):
break
time.sleep(0.1)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
time.sleep(0.1)
assert not os.path.exists(
path1), 'service foo does exist or is not a directory'
assert not os.path.exists(
path2), 'service bar does exist or is not a directory'
assert os.path.exists(
path3), 'service agent does NOT exist or is not a directory'
assert not testThFoo.isAlive(), 'thread Foo is still alive'
assert not testThBar.isAlive(), 'thread Bar is still alive'
assert not testThAgent.isAlive(), 'thread Agent is still alive'
assert not os.path.exists(pkgPath), 'package foo exists'
assert not os.path.exists(pkgPropPath), 'package prop foo exists'
assert os.path.exists(pkgDir), 'package directory does not exist'
# ensure agent cleanup is proper
active_mf_path = ManifestController.manifestPath('agent', active_mf)
active_link = os.path.join(ServiceController.manifestPath('agent'),
'active')
all_mf.remove(active_mf)
actual_active_mf_path = utils.readlink(active_link)
self.assertTrue(
os.path.exists(active_mf_path),
'active agent manifest got deleted but shouldn\t have')
self.assertTrue(os.path.exists(active_link),
'agent active link missing')
self.assertEqual(
active_mf_path, actual_active_mf_path,
'agent active link pointing to some wrong manifest; link broken?')
for mf in all_mf:
agnt_mf_path = ManifestController.manifestPath('agent', mf)
self.assertFalse(
os.path.exists(agnt_mf_path),
'non active agent mf %s should have been deleted' % mf)
def _addMonitors(self, service, activeManifest):
''' add monitors for a service's activeManifest '''
monitorTasks = []
self.__monitorTasks[(service, activeManifest)] = monitorTasks
# metric tags
metricTags = {}
# add default metric tags
metricTags['fqdn'] = utils.fqdn
# read tags from agent
agtMetricTags = manifestutil.readJsonServiceMeta('agent', [METRIC_TAGS_KEY])
if agtMetricTags and METRIC_TAGS_KEY in agtMetricTags:
metricTags.update(agtMetricTags[METRIC_TAGS_KEY])
# read tags from service
if (service != 'agent'):
serviceMetricTags = manifestutil.readJsonServiceMeta(service, [METRIC_TAGS_KEY])
if serviceMetricTags or METRIC_TAGS_KEY in serviceMetricTags:
metricTags.update(serviceMetricTags[METRIC_TAGS_KEY])
# all packages in this manifest
packages = ManifestController.getPackages(service, os.path.basename(activeManifest))
# metric tags for externally passed in metrics
# monitorTags = {additionaltags_dict, datatype_dict, ctx(env.pool.host), appType(Monitor|custom), monitorType(Cronus|CronusApplication)}
self.__monitorTags[service] = {'default': (metricTags, {}, {})}
self.__monitorValues[(service, 'default')] = {}
self.__monitorMessages[(service, 'default')] = {}
# a unique index for all monitors in the service
for package in packages:
# load monitor setting from cronus.ini
pkgConfigs = manifestutil.getPackageInitConfig(service, activeManifest, package)
if pkgConfigs is not None and 'monitors' in pkgConfigs:
metricCtx = pkgConfigs[METRIC_CONTEXT] if METRIC_CONTEXT in pkgConfigs else {}
metricTypes = {}
if METRIC_TYPE_KEY in pkgConfigs and pkgConfigs[METRIC_TYPE_KEY]:
metricTypes = pkgConfigs[METRIC_TYPE_KEY]
if METRIC_TAGS_KEY in pkgConfigs and pkgConfigs[METRIC_TAGS_KEY]:
metricTags.update(pkgConfigs[METRIC_TAGS_KEY])
monitors = pkgConfigs['monitors']
for monitor in monitors:
try:
m_name = str(monitor['name'])
# metric tags for externally passed in metrics
# monitorTags = {additionaltags_dict, datatype_dict, ctx(env.pool.host), appType(Monitor|custom), monitorType(Cronus|CronusApplication)}
self.__monitorTags[service][m_name] = (metricTags, metricTypes, metricCtx)
# now normalize the interval to one allowable by EVPS (downcast)
if 'reportIntervalSec' not in monitor:
continue
m_interval = float(monitor['reportIntervalSec'])
m_interval = self._normalizeResolution(m_interval)
m_timeout = float(monitor['timeoutSec']) if 'timeoutSec' in monitor else m_interval
m_timeout = max(1, min(m_interval-1, m_timeout))
m_items = monitor['items'] if 'items' in monitor else []
# prime monitor value store
self.__monitorValues[(service, m_name)] = {}
self.__monitorMessages[(service, m_name)] = {}
mi_idx = 0
for m_item in m_items:
if 'type' not in m_item:
continue
mi_idx += 1
mi_type = m_item['type']
mi_cmd = None
if 'script' == mi_type:
mi_script = m_item['script']
mi_cmd = manifestutil.getMonitorScriptPath(service, activeManifest, package, mi_script)
else:
# unknown type, skip
LOG.error('Unknown monitor item type %s, can only be script' % mi_type)
continue
# schedule monitor
m_sch_name = '%s.%s' % (service, m_name)
try:
self.__monitorMessages[(service, m_name)][mi_cmd] = []
task = self.__monitorSch.add_interval_task(self._runMonitor, '%s.%s' % (m_sch_name, str(mi_idx)), 0, float(m_interval),
[mi_type,mi_cmd,service,m_name,m_timeout,(str(int(round(m_interval))),metricTags,metricTypes,metricCtx)],None)
monitorTasks.append(task)
except Exception as excep:
LOG.error('Cannot add monitor task for %s.%s - (%s) %s' % (m_sch_name, str(mi_idx), excep, traceback.format_exc(5)))
except Exception as excep1:
LOG.error('Cannot add report task for %s.%s - (%s) %s' % (service, m_name, excep1, traceback.format_exc(5)))
