def test_cleanupOrphanedPackages2(self):
from agent.lib.utils import rchown
from agent.tests.unit.test_util import createManifest
serviceName = 'service2'
manifestName = 'manifestA'
serviceName1 = 'service3'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"],
service = serviceName, manifest = manifestName)
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"],
service = serviceName1, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
installedPkgPath = os.path.realpath(os.path.join(manifestutil.serviceRootPath() , serviceName, 'installed-packages'))
installedPkgPath1 = os.path.realpath(os.path.join(manifestutil.serviceRootPath() , serviceName1, 'installed-packages'))
uname = configutil.getAgentUser()
uid, gid = utils.getUidGid(uname)
rchown(installedPkgPath, uid, gid)
rchown(installedPkgPath1, uid, gid)
self.assertEquals(len(os.listdir(installedPkgPath)) , 1)
self.assertEquals(len(os.listdir(installedPkgPath1)) , 1)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName1, manifest = manifestName), expect_errors = True)
pylons.config['packageMgr_install_package_age'] = 0
pylons.config['packageMgr_install_package_min_age'] = 0.0
PackageMgr.cleanupOrphanedPackages()
self.assertEquals(len(os.listdir(installedPkgPath)) , 0)
self.assertEquals(len(os.listdir(installedPkgPath1)) , 0)
def _untarPackages(self, packages, service, untarRootPath, nicelevel=None, pgksNeedSuffix=None, pathSuffix=''):
"""
untar all the packages
@params packages - list of dictionary of parsed packages uri
@throws AgentException
"""
self._updateProgress(80)
count = 0
for pkgDict in packages.itervalues():
count += 1
# check if package path exists already
# if the package is already untarred, move on
# else create the package path
pkgName = pkgDict['packageName']
untarPath = os.path.join(untarRootPath,
pkgName,
'%s%s' % (pkgDict['packageVersion'], pathSuffix if ((pgksNeedSuffix is not None) and pkgName in pgksNeedSuffix) else ''))
if os.path.exists(untarPath):
# perhaps another thread has finished extraction, continue with next package
continue
os.makedirs(untarPath)
try:
self.untar(pkgDict['packagePath'], untarPath, nicelevel)
except AgentException:
rmrf(untarPath)
# Note: i. atleast self should have 'rx' so that we can proceed setting 'rx' for group and others
# ii. since cronus/cronusapp belong to diff group, lets give access to self, group and others.
# if both belong to same group in future, then just self, group should be enough
if (not os.name == 'nt'):
# ensure all parent dir of scripts dir have 'rx' so that we can really navigate to scripts dir and execute
uname = getuserofpath(untarPath)
chmod(untarPath, '+rx', sudoUser = uname)
cronusPath = os.path.join(untarPath, 'cronus')
if os.path.exists(cronusPath):
uname = getuserofpath(cronusPath)
chmod(cronusPath, '+rx', sudoUser = uname)
# now give all scripts 'rx' permission
scriptsPath = os.path.join(cronusPath, 'scripts')
if os.path.exists(scriptsPath):
uname = getuserofpath(scriptsPath)
rchmod(scriptsPath, '+rx', sudoUser = uname)
#Running as cronus user when higher privilege service (i.e. not chown all the package into the application user)
if (not isHigherPrivilegeService(service)):
import pwd
uname = pylons.config['app_user_account']
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
rchown(untarPath, uid, gid)
self._updateProgress(calcProgress(80, 99, float(count) / len(packages)))
def test_cleanUpAllInstalledPackages(self):
from agent.lib.utils import rchown
from agent.tests.unit.test_util import createManifest
#import pdb;pdb.set_trace();
serviceName = 'service3'
manifestName = 'manifestB'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
installedPkgPath = os.path.realpath(os.path.join(manifestutil.serviceRootPath() , serviceName, 'installed-packages'))
uname = configutil.getAgentUser()
uid, gid = utils.getUidGid(uname)
rchown(installedPkgPath, uid, gid)
pkgs = []
for pkg in os.listdir(installedPkgPath):
pkgs.append(os.path.join(installedPkgPath, pkg))
pkgVers = []
for pkg in pkgs:
for pkgVer in os.listdir(pkg):
pkgVers.append(os.path.join(pkg, pkgVer))
self.assertEquals(len(os.listdir(installedPkgPath)) , 1)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
#import pdb; pdb.set_trace();
symLinks = [] #ManifestController.getAllSymLinks(serviceName)
orphans = set(pkgVers) - set(symLinks)
age = pylons.config['packageMgr_install_package_age']
minage = pylons.config['packageMgr_install_package_min_age']
pylons.config['packageMgr_install_package_age'] = 0
pylons.config['packageMgr_install_package_min_age'] = 0.0
self.assertEquals(len(os.listdir(installedPkgPath)) , 1)
PackageUtil.cleanupInstalledPkgs(installedPkgPath, orphans)
#PackageUtil.cleanUpInstalledPkgs(installedPkgPath, orphans)
self.assertEquals(len(os.listdir(installedPkgPath)) , 0)
pylons.config['packageMgr_install_package_age'] = age
pylons.config['packageMgr_install_package_min_age'] = minage
def testDelete(self):
path = os.path.join(pylons.config['agent_root'], 'service_nodes', 'foo')
LOG.debug('************ path = %s' % path)
LOG.debug('************ first delete')
response = self.app.delete(url(controller='service', service='foo', action='delete'), expect_errors = True)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 500, "HTTP response != 500"
os.makedirs(path)
os.makedirs(os.path.join(path, 'manifest'))
os.makedirs(os.path.join(path, 'installed-packages'))
uname = configutil.getAgentUser()
uid, gid = utils.getUidGid(uname)
rchown(path, uid, gid)
LOG.debug('************ second delete')
response = self.app.delete(url(controller='service', service='foo', action='delete'))
LOG.debug ('Delete response body = ' + response.body)
body = json.loads(response.body)
tm = time.time()
while (tm + 5 > time.time()):
response = self.app.get(body['status'], expect_errors = True)
LOG.debug ('Status response body = ' + response.body)
body = json.loads(response.body)
if (body['progress'] == 100):
break
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
assert not os.path.exists(path), 'service foo does exist or is not a directory'
def testDelete2(self):
createManifest(self)
# need to change the owner of the service dir to be cronus
servicePath = ServiceController.servicePath('foo')
uname = configutil.getAgentUser()
uid, gid = utils.getUidGid(uname)
rchown(servicePath, uid, gid)
response = self.app.delete(url(controller='service', service='foo', action='delete'))
assert response.status_int == 200, "HTTP response != 200"
body = json.loads(response.body)
tm = time.time()
while (tm + 5 > time.time()):
response = self.app.get(body['status'], expect_errors = True)
LOG.debug ('************* Status response body = ' + response.body)
body = json.loads(response.body)
if (body['progress'] == 100):
break
assert not os.path.isdir(ServiceController.servicePath('foo'))
def _untarPackages(self, packages, service, untarRootPath, nicelevel=None, pgksNeedSuffix=None, pathSuffix=''):
"""
untar all the packages
@params packages - list of dictionary of parsed packages uri
@throws AgentException
"""
self._updateProgress(80)
count = 0
for pkgDict in packages.itervalues():
count += 1
# check if package path exists already
# if the package is already untarred, move on
# else create the package path
pkgName = pkgDict['packageName']
untarPath = os.path.join(untarRootPath,
pkgName,
'%s%s' % (pkgDict['packageVersion'], pathSuffix if ((pgksNeedSuffix is not None) and pkgName in pgksNeedSuffix) else ''))
if os.path.exists(untarPath):
# perhaps another thread has finished extraction, continue with next package
continue
os.makedirs(untarPath)
try:
self.untar(pkgDict['packagePath'], untarPath, nicelevel)
except AgentException:
rmrf(untarPath)
# Note: i. atleast self should have 'rx' so that we can proceed setting 'rx' for group and others
# if both belong to same group in future, then just self, group should be enough
# ensure all parent dir of scripts dir have 'rx' so that we can really navigate to scripts dir and execute
uname = getuserofpath(untarPath)
chmod(untarPath, '+rx', sudoUser = uname)
cronusPath = os.path.join(untarPath, 'cronus')
if os.path.exists(cronusPath):
uname = getuserofpath(cronusPath)
chmod(cronusPath, '+rx', sudoUser = uname)
# now give all scripts 'rx' permission
scriptsPath = os.path.join(cronusPath, 'scripts')
if os.path.exists(scriptsPath):
uname = getuserofpath(scriptsPath)
rchmod(scriptsPath, '+rx', sudoUser = uname)
# issue #16, now add symlink to .appdata for easy access
appdata_path = manifestutil.appDataPath(service)
link_path = os.path.join(untarPath, '.appdata')
LOG.info('Create .appdata symlink from %s to %s' % (link_path, appdata_path))
utils.symlink(appdata_path, link_path)
#Running as cronus user when higher privilege service (i.e. not chown all the package into the application user)
if (not isHigherPrivilegeService(service)):
uname = configutil.getAppUser()
uid, gid = utils.getUidGid(uname)
rchown(untarPath, uid, gid)
self._updateProgress(calcProgress(80, 99, float(count) / len(packages)))
def doRun(self):
""" Main body of the thread """
spath = manifestutil.servicePath(self._service)
self._updateProgress(1)
errorMsg = ""
errorCode = None
failed = False
ctxNames = ['guid', 'service']
try:
# create service if not already exist
if not os.path.exists(spath):
os.makedirs(spath)
os.makedirs(os.path.join(spath, 'manifests'))
os.makedirs(os.path.join(spath, 'installed-packages'))
os.makedirs(os.path.join(spath, 'modules'))
os.makedirs(os.path.join(spath, 'downloaded-packages'))
os.makedirs(os.path.join(spath, '.appdata'))
os.makedirs(os.path.join(spath, '.data'))
import pwd
uname = pylons.config['app_user_account']
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
utils.rchown(os.path.join(spath, '.appdata'), uid, gid)
# verify that the path exists
if (not os.path.isdir(spath)):
raise AgentException(Errors.UNKNOWN_ERROR, "Service(%s) was not created" % self._service)
self._updateProgress(20)
# create manifest if not already exist
mpath = manifestutil.manifestPath(self._service, self._manifest)
if (not os.path.exists(mpath) or not os.path.isdir(mpath)):
self.__LOG.debug('pkgs = %s', self.__packages)
# parse the package list
for idx, package in enumerate(self.__packages):
if package.startswith('/'):
packageRef = package
tokens = package.split('/')
pkgnamePrefix = tokens[-1].rstrip()
fullPkgLoc = manifestutil.getPackageByName(self._service, manifest = None, pkgnamePrefix = pkgnamePrefix)
if fullPkgLoc is None:
raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
'manifest (%s/%s) package (%s) does not exist' %
(self._service, self._manifest, self.__packages))
else:
self.__LOG.info('expanding package reuse ref %s with full package location %s' % (packageRef, fullPkgLoc))
self.__packages[idx] = fullPkgLoc
# start a thread to create the package
manThread = ManifestCreate(threadmgr.NULL_THREADMGR, self._service, self._manifest, self.__packages, skipProp = self.__skipProp)
contextutils.copycontexts(self, manThread, ctxNames)
manThread.run()
status = manThread.getStatus()
if (status['error'] != None):
raise AgentException(status['error'], status['errorMsg'])
self._updateProgress(60)
if (not os.path.exists(mpath)):
raise AgentException(Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Manifest(%s, %s) path missing' % (self._service, self._manifest))
if not self.__skipActivation:
activateThread = ActivateManifest(threadmgr.NULL_THREADMGR, self._service, self._manifest)
contextutils.copycontexts(self, activateThread, ctxNames)
activateThread.run()
status = activateThread.getStatus()
if (status['error'] != None):
raise AgentException(status['error'], status['errorMsg'])
# activte manifest if not already activated
if manifestutil.getActiveManifest(self._service) != self._manifest:
raise AgentException(Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Manifest(%s, %s) path missing' % (self._service, self._manifest))
except SystemExit as exc:
failed = True
if (len(exc.args) == 2):
# ok we got {err code, err msg}
errorCode = exc.args[0]
errorMsg = exc.args[1]
except AgentException as exc:
failed = True
errorMsg = 'Deploy Service - Agent Exception - %s' % exc.getMsg()
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Deploy Service - Unknown error - (%s/%s) - %s - %s' \
% (self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.warning(errorMsg)
self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
else:
self._updateProgress(100)
def rchown(path, uname):
uid, gid = utils.getUidGid(uname)
rchown(path, uid, gid)
def startServicesOnAgentStartup():
"""
when agent is restarted,
0. check for agent selfupdate
1. start all service with active manifest, this requires service startup script be idempotent
2. load dynamic controllers and routes if any
"""
# check for agent update
from agent.lib.agenthealth import checkAgentVersion
checkAgentVersion(True)
# startup services
from agent.lib.agent_thread.startstop_service import StartStopService
appGlobal = config["pylons.app_globals"]
appdelay = int(config["appinitdelay"]) if "appinitdelay" in config else 0
if appdelay > 0:
time.sleep(appdelay)
# check if this is agent restart or system restart
if os.name != "nt" and os.path.exists("/proc/uptime"):
uptime, _ = [float(f) for f in open("/proc/uptime").read().split()]
else:
uptime = 500
systemRestartTimeThreshold = pylons.config["system_restart_time_threshold"]
actionType = StartStopService.ACTION_RESTART
if int(systemRestartTimeThreshold) > uptime:
actionType = StartStopService.ACTION_REBOOT
for service in manifestutil.getServices():
appDataDir = manifestutil.appDataPath(service)
if not os.path.exists(appDataDir):
os.makedirs(appDataDir)
import pwd
uname = pylons.config["app_user_account"]
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
utils.rchown(appDataDir, uid, gid)
dataDir = manifestutil.dataPath(service)
if not os.path.exists(dataDir):
os.makedirs(dataDir)
if service != "agent":
try:
manifestutil.updateServiceMetaFile(
service, {"servicePath": manifestutil.servicePath(service), "serviceName": service}
)
except Exception as excep:
LOG.error(
"Unknown error updating local metadata service(%s) - %s - %s"
% (service, str(excep), traceback.format_exc(2))
)
if manifestutil.hasActiveManifest(service):
try:
LOG.info("startup for service(%s)", service)
startupThread = StartStopService(appGlobal.threadMgr, service, actionType)
startupThread.start()
startupThread.threadMgrEvent.wait()
except Exception as excep:
LOG.error(
"Unknown error starting service(%s) - %s - %s"
% (service, str(excep), traceback.format_exc(2))
)
try:
LOG.info("load controllers and routes for service(%s)", service)
manifestutil.processControllerInPackage(service)
except Exception as excep:
LOG.error(
"Unknown error loading controllers for service(%s) - %s - %s"
% (service, str(excep), traceback.format_exc(2))
)
def rchown(path, uname):
import pwd
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
rchown(path, uid, gid)
