def createManifests(mf_path):
os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
latest = os.path.join(mf_path, 'm3.0')
os.makedirs(os.path.join(latest, 'dummy_dir3.0'))
utils.symlink(latest, os.path.join(mf_path, 'active'))
return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')
def createManifests(mf_path):
os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
latest = os.path.join(mf_path, 'm3.0')
os.makedirs(os.path.join(latest, 'dummy_dir3.0'))
utils.symlink(latest, os.path.join(mf_path, 'active'))
return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')
def testValidateLink(self):
commonTearDown()
commonSetup()
basepath = pylons.config['agent_root']
badpath = os.path.join(basepath, 'not_exist')
goodpath = os.path.join(basepath, 'packages')
badlink = os.path.join(basepath, 'bad_link')
goodlink = os.path.join(basepath, 'good_link')
utils.symlink(badpath, badlink)
assert (utils.validatelink(badlink) == False)
utils.symlink(goodpath, goodlink)
assert (utils.validatelink(goodlink) == True)
def __switchSymlink(self, service, manifest):
""" remove old symlink and create new one """
self._LOG.info("Switch active link for %s-%s" % (service, manifest))
if (manifest == None):
return
#remove symlink
activePath = self.__getSymlinkPath(service)
self.__removeSymlink(activePath)
# create the active link
symlink(manifest, activePath)
return True
def testValidateLink(self):
commonTearDown()
commonSetup()
basepath = pylons.config['agent_root']
badpath = os.path.join(basepath, 'not_exist')
goodpath = os.path.join(basepath, 'packages')
badlink = os.path.join(basepath, 'bad_link')
goodlink = os.path.join(basepath, 'good_link')
utils.symlink(badpath, badlink)
assert(utils.validatelink(badlink) == False)
utils.symlink(goodpath, goodlink)
assert(utils.validatelink(goodlink) == True)
def __switchSymlink(self, service, manifest):
""" remove old symlink and create new one """
self.__LOG.debug("Switch active link for %s-%s" % (service, manifest))
if (manifest == None):
return
#remove symlink
activePath = self.__getSymlinkPath(service)
self.__removeSymlink(activePath)
# create the active link
symlink(manifest, activePath)
return True
def test_delete_active_manifest(self):
createManifest(self)
self.assertTrue(os.path.isdir(ServiceController.manifestPath('foo')))
currentPath = os.getcwd()
os.chdir(ServiceController.manifestPath('foo'))
manifestPath = 'bar' #ManifestController.manifestPath('foo', 'bar'); use short manifest name instead of full path
symlink(manifestPath, 'active')
os.chdir(currentPath)
response = self.app.delete(url(controller = 'manifest', action = 'delete', service = "foo", manifest = "bar"),
expect_errors = True)
self.assertEquals(500, response.status_int)
body = json.loads(response.body)
self.assertEquals(Errors.MANIFEST_DELETING_ACTIVE_MANIFEST, body['error'])
def test_delete_active_manifest(self):
createManifest(self)
self.assertTrue(os.path.isdir(ServiceController.manifestPath('foo')))
currentPath = os.getcwd()
os.chdir(ServiceController.manifestPath('foo'))
manifestPath = 'bar' #ManifestController.manifestPath('foo', 'bar'); use short manifest name instead of full path
symlink(manifestPath, 'active')
os.chdir(currentPath)
response = self.app.delete(url(controller = 'manifest', action = 'delete', service = "foo", manifest = "bar"),
expect_errors = True)
self.assertEquals(500, response.status_int)
body = json.loads(response.body)
self.assertEquals(Errors.MANIFEST_DELETING_ACTIVE_MANIFEST, body['error'])
def testGet(self):
path = manifestutil.manifestPath('foo', 'bar')
os.makedirs(path)
path = manifestutil.manifestPath('foo', 'baz')
os.makedirs(path)
activePath = os.path.join(ServiceController.manifestPath('foo'), 'active')
symlink('bar', activePath)
response = self.app.get(url(controller='service', service='foo', action='get'), expect_errors = True)
body = json.loads(response.body)
print "************** response = %s" % body
assert body['progress'] == 100
assert body['result']['activemanifest'] == 'bar'
assert body['result']['manifest'] == ['bar', 'baz']
def testGet(self):
path = ManifestController.manifestPath('foo', 'bar')
os.makedirs(path)
path = ManifestController.manifestPath('foo', 'baz')
os.makedirs(path)
activePath = os.path.join(ServiceController.manifestPath('foo'),
'active')
symlink('bar', activePath)
response = self.app.get(url(controller='service',
service='foo',
action='get'),
expect_errors=True)
body = json.loads(response.body)
print "************** response = %s" % body
assert body['progress'] == 100
assert body['result']['activemanifest'] == 'bar'
assert body['result']['manifest'] == ['bar', 'baz']
def doRun(self):
""" Main body of the thread
Progress Info:
Look at the list of packages this manifest has and figure out the progress of this manifest
Progress 1 = manifest directory created
Progress 2-80 = all packages downloaded
Progress 81-99 = all packages untarred
Progress 100 = all links done
"""
inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(self.__service, self.__manifest))
try:
if self.__service != 'agent':
utils.checkDiskFull()
installedPkgPath = ServiceController.installedPkgPath(self.__service)
# This shouldn'trn happen but make sure there isn'trn already a manifest directory in progress
if (os.path.isdir(inProgressPath)):
# probably another manifest create thread died out half way.
# Cleanup and reattempt manifest creation
LOG.debug('Manifest inprogress found for service/manifest (%s/%s). Will cleanup and retry' % (self.__service, self.__manifest))
shutil.rmtree(inProgressPath)
# make sure that the service path and installed package path exists
if (not os.path.isdir(ServiceController.manifestPath(self.__service)) or
not os.path.isdir(ServiceController.installedPkgPath(self.__service))):
errCode = Errors.SERVICE_NOT_FOUND
msg = 'Service (%s) for manifest (%s) not found' % (self.__service, self.__manifest)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
return
# ok create the manifest path
os.mkdir(inProgressPath)
self._updateProgress(1)
# figure out which of the packages are already there
remainingPackages = {}
for pkgUri in self.__packages:
pkgDict = PackageUtil.parseUri(pkgUri)
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
pkgName = pkgDict['packageName']
if (not os.path.exists(pkgPath)) or ((self.__forcePackages is not None) and pkgName in self.__forcePackages):
remainingPackages[pkgUri] = pkgDict
else:
symlink(pkgPath, os.path.join(inProgressPath, pkgDict['packageName']))
if self.__attemptDownload:
# now make sure all the packages are downloaded
try:
self._downloadPackages(remainingPackages.keys(), skipProp = self.__skipProp)
except AgentException as exc:
# check if it is download error, then modify exception appropriately
if exc.getCode() == Errors.DC_FAILED_DOWNLOAD:
exc = AgentException(Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED,
'Manifest (%s/%s) failed downloading package - %s'
% (self.__service, self.__manifest, exc.getMsg()))
raise exc
else:
if len(remainingPackages) > 0:
raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
'Create Manifest (%s/%s) failed since package is not present and download has been disabled'
% (self.__service, self.__manifest))
LOG.info('Completed download all packages for (%s/%s)' % (self.__service, self.__manifest))
# now untar the packages
import re
pkgSuffix = '.%s' % re.sub(r"\W", "", self.__manifest)
self._untarPackages(remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service), 0, self.__forcePackages, pkgSuffix)
LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__manifest))
# now create the links
for pkgDict in remainingPackages.itervalues():
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
linkPath = os.path.join(inProgressPath, pkgDict['packageName'])
# validate target folder does exist
if not os.path.exists(pkgPath):
raise AgentException(Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath)
symlink(pkgPath, linkPath)
# now move the inProgressPath to the final path
manifestContentPath = ManifestController.manifestContentPath(self.__service, self.__manifest)
os.rename(inProgressPath, ManifestController.manifestPath(self.__service, self.__manifest))
mfContentsFile = file(manifestContentPath, 'w')
for pkgUri in self.__packages:
mfContentsFile.write(('%s%s') % (pkgUri, os.linesep))
mfContentsFile.close()
LOG.info('Completed create manifest directories for (%s/%s)' % (self.__service, self.__manifest))
LOG.info('Completed create manifest for (%s/%s)' % (self.__service, self.__manifest))
self._updateStatus(progress = 100)
except AgentException as exc:
LOG.info(exc.getMsg())
self._updateStatus(httpStatus = 500, error = exc.getCode(), errorMsg = exc.getMsg())
except Exception as exc:
errCode = Errors.UNKNOWN_ERROR
msg = 'Unknown error for (%s/%s) - %s - %s' % (self.__service, self.__manifest,
str(exc), traceback.format_exc(2))
LOG.info(msg)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
finally:
# clean up intermediate progress
try:
shutil.rmtree(inProgressPath)
except OSError:
pass
def doRun(self):
""" Main body of the thread
Progress Info:
Look at the list of packages this manifest has and figure out the progress of this manifest
Progress 1 = manifest directory created
Progress 2-80 = all packages downloaded
Progress 81-99 = all packages untarred
Progress 100 = all links done
"""
try:
if self.__service != 'agent':
utils.checkDiskFull()
installedPkgPath = manifestutil.installedPkgRootPath(self.__service)
# figure out which of the packages are already there
remainingPackages = {}
pkgDict = PackageUtil.parseUri(self.__package)
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
remainingPackages[self.__package] = pkgDict
if (not os.path.exists(pkgPath)):
# now make sure all the packages are downloaded
try:
self._downloadPackages([self.__package])
except AgentException as exc:
# check if it is download error, then modify exception appropriately
if exc.getCode() == Errors.DC_FAILED_DOWNLOAD:
exc = AgentException(Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED,
'Manifest (%s/%s) failed downloading package - %s'
% (self.__service, self.__module, exc.getMsg()))
raise exc
LOG.info('Completed download packages for (%s/%s)' % (self.__service, self.__module))
# now untar the packages
self._untarPackages(remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service))
LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__module))
# now create the links
for pkgDict in remainingPackages.itervalues():
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
modulePath = os.path.join(manifestutil.moduleRootPath(self.__service), self.__module)
# validate target folder does exist
if not os.path.exists(pkgPath):
raise AgentException(Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath)
# remove existing module
isExistingModule = os.path.exists(modulePath)
if isExistingModule:
execThread = self._getBuiltThread('deactivate')
if execThread:
super(ModuleCreate, self)._runExeThread(execThread)
rmlink(modulePath)
execThread = self._getBuiltThread('activate')
if execThread:
super(ModuleCreate, self)._runExeThread(execThread)
symlink(pkgPath, modulePath)
# load controllers from package
manifestutil.processModule(self.__service, self.__module, not isExistingModule)
LOG.info('Completed create module for (%s/%s)' % (self.__service, self.__module))
self._updateStatus(progress = 100)
except AgentException as exc:
LOG.info(exc.getMsg())
self._updateStatus(httpStatus = 500, error = exc.getCode(), errorMsg = exc.getMsg())
except Exception as exc:
errCode = Errors.UNKNOWN_ERROR
msg = 'Unknown error for (%s/%s) - %s - %s' % (self.__service, self.__module,
str(exc), traceback.format_exc(2))
LOG.info(msg)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
def doRun(self):
""" Main body of the thread
Progress Info:
Look at the list of packages this manifest has and figure out the progress of this manifest
Progress 1 = manifest directory created
Progress 2-80 = all packages downloaded
Progress 81-99 = all packages untarred
Progress 100 = all links done
"""
try:
if self.__service != 'agent':
utils.checkDiskFull()
installedPkgPath = manifestutil.installedPkgRootPath(
self.__service)
# figure out which of the packages are already there
remainingPackages = {}
pkgDict = PackageUtil.parseUri(self.__package)
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'],
pkgDict['packageVersion'])
remainingPackages[self.__package] = pkgDict
if (not os.path.exists(pkgPath)):
# now make sure all the packages are downloaded
try:
self._downloadPackages([self.__package])
except AgentException as exc:
# check if it is download error, then modify exception appropriately
if exc.getCode() == Errors.DC_FAILED_DOWNLOAD:
exc = AgentException(
Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED,
'Manifest (%s/%s) failed downloading package - %s'
% (self.__service, self.__module, exc.getMsg()))
raise exc
LOG.info('Completed download packages for (%s/%s)' %
(self.__service, self.__module))
# now untar the packages
self._untarPackages(
remainingPackages, self.__service,
ServiceController.installedPkgPath(self.__service))
LOG.info('Completed untar all packages for (%s/%s)' %
(self.__service, self.__module))
# now create the links
for pkgDict in remainingPackages.itervalues():
pkgPath = os.path.join(installedPkgPath,
pkgDict['packageName'],
pkgDict['packageVersion'])
modulePath = os.path.join(
manifestutil.moduleRootPath(self.__service), self.__module)
# validate target folder does exist
if not os.path.exists(pkgPath):
raise AgentException(
Errors.PACKAGE_PATH_ERROR,
'invalid untarred package at %s' % pkgPath)
# remove existing module
isExistingModule = os.path.exists(modulePath)
if isExistingModule:
execThreadTuple = self._getBuiltThread('deactivate')
super(ModuleCreate,
self)._runExeThread(self, execThreadTuple)
rmlink(modulePath)
execThreadTuple = self._getBuiltThread('activate')
super(ModuleCreate, self)._runExeThread(execThreadTuple)
symlink(pkgPath, modulePath)
# load controllers from package
manifestutil.processModule(self.__service, self.__module,
not isExistingModule)
LOG.info('Completed create module for (%s/%s)' %
(self.__service, self.__module))
self._updateStatus(progress=100)
except AgentException as exc:
LOG.info(exc.getMsg())
self._updateStatus(httpStatus=500,
error=exc.getCode(),
errorMsg=exc.getMsg())
except Exception as exc:
errCode = Errors.UNKNOWN_ERROR
msg = 'Unknown error for (%s/%s) - %s - %s' % (
self.__service, self.__manifest, str(exc),
traceback.format_exc(2))
LOG.info(msg)
self._updateStatus(httpStatus=500, error=errCode, errorMsg=msg)
def _untarPackages(self, packages, service, untarRootPath, nicelevel=None, pgksNeedSuffix=None, pathSuffix=''):
"""
untar all the packages
@params packages - list of dictionary of parsed packages uri
@throws AgentException
"""
self._updateProgress(80)
count = 0
for pkgDict in packages.itervalues():
count += 1
# check if package path exists already
# if the package is already untarred, move on
# else create the package path
pkgName = pkgDict['packageName']
untarPath = os.path.join(untarRootPath,
pkgName,
'%s%s' % (pkgDict['packageVersion'], pathSuffix if ((pgksNeedSuffix is not None) and pkgName in pgksNeedSuffix) else ''))
if os.path.exists(untarPath):
# perhaps another thread has finished extraction, continue with next package
continue
os.makedirs(untarPath)
try:
self.untar(pkgDict['packagePath'], untarPath, nicelevel)
except AgentException:
rmrf(untarPath)
# Note: i. atleast self should have 'rx' so that we can proceed setting 'rx' for group and others
# if both belong to same group in future, then just self, group should be enough
# ensure all parent dir of scripts dir have 'rx' so that we can really navigate to scripts dir and execute
uname = getuserofpath(untarPath)
chmod(untarPath, '+rx', sudoUser = uname)
cronusPath = os.path.join(untarPath, 'cronus')
if os.path.exists(cronusPath):
uname = getuserofpath(cronusPath)
chmod(cronusPath, '+rx', sudoUser = uname)
# now give all scripts 'rx' permission
scriptsPath = os.path.join(cronusPath, 'scripts')
if os.path.exists(scriptsPath):
uname = getuserofpath(scriptsPath)
rchmod(scriptsPath, '+rx', sudoUser = uname)
# issue #16, now add symlink to .appdata for easy access
appdata_path = manifestutil.appDataPath(service)
link_path = os.path.join(untarPath, '.appdata')
LOG.info('Create .appdata symlink from %s to %s' % (link_path, appdata_path))
utils.symlink(appdata_path, link_path)
#Running as cronus user when higher privilege service (i.e. not chown all the package into the application user)
if (not isHigherPrivilegeService(service)):
uname = configutil.getAppUser()
uid, gid = utils.getUidGid(uname)
rchown(untarPath, uid, gid)
self._updateProgress(calcProgress(80, 99, float(count) / len(packages)))
