def test_post_manifest_inprogress_errorout(self):
packages = ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']
manifest = 'bar'
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response.status_int == 200, 'Manifest Post assert'
try:
response = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
#should error out!
raise Exception('Oops! Expected AppError but did not get any')
except AppError:
pass
def test_multiple_manifest_create(self):
""" when a manifest creation is in progress for a service, another creation should block """
packages = ["http://repository.qa.ebay.com/cronus/test-data/agent/pkgA-1.2.0.unix.cronus"]
service = 'foo'
manifest1 = 'bar'
manifest2 = 'car'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
self.assertEquals(response1.status_int, 200, 'Manifest1 Post assert - should go through')
try:
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
self.assertFalse(True, 'Expected an exception but did not get one!')
except AppError:
pass
checkStatus(self, 'create manifest bar', response1, timeout = 25)
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'pkgA')))
self.assertFalse(islink(os.path.join(ManifestController.manifestPath('foo', 'car'), 'pkgA')))
def test_multiple_manifest_create(self):
""" when a manifest creation is in progress for a service, another creation should block """
packages = ["http://www.stackscaling.com/agentrepo/pkgA-1.2.0.unix.cronus"]
service = 'foo'
manifest1 = 'bar'
manifest2 = 'car'
try:
for pkg in packages:
mockDownloadPkg(pkg)
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
self.assertEquals(response1.status_int, 200, 'Manifest1 Post assert - should go through')
try:
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
self.assertFalse(True, 'Expected an exception but did not get one!')
except AppError:
pass
checkStatus(self, 'create manifest bar', response1, timeout = 25)
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'pkgA')))
self.assertFalse(islink(os.path.join(manifestutil.manifestPath('foo', 'car'), 'pkgA')))
def test_post_manifest_inprogress_ok(self):
service = 'foo'
manifest = 'blahblah'
try:
path = ServiceController.servicePath(service)
if os.path.exists(path):
if os.name == 'nt':
cmd = 'rm -r %s' % path
LOG.debug("running command %s" % cmd)
os.system(cmd)
else:
shutil.rmtree(path)
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(service, manifest))
os.makedirs(inProgressPath)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus"], manifest = 'blahblah', createDirs = False)
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'blahblah'), 'pkgA')))
def test_post_manifest_inprogress_errorout(self):
packages = ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']
manifest = 'bar'
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response.status_int == 200, 'Manifest Post assert'
try:
response = self.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
#should error out!
raise Exception('Oops! Expected AppError but did not get any')
except AppError:
pass
def test_post2(self):
# successful post
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def createManifest(testController,
packages = ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus'],
manifest = 'bar', service = 'foo', createDirs = True):
try:
path = ServiceController.servicePath(service)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'modules'))
os.makedirs(os.path.join(path, '.appdata'))
os.makedirs(os.path.join(path, '.data'))
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
try:
for pkgUri in packages:
mockDownloadPkg(pkgUri)
# pkgDict = PackageUtil.parseUri(pkgUri)
# pkgName = pkgDict['package']
# localPkgRoot = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'packages')
# src = os.path.join(localPkgRoot, pkgName)
# shutil.copy(src, pkgDict['packagePath'])
# propPkgName = pkgDict['propName']
# propSrc = os.path.join(localPkgRoot, propPkgName)
# shutil.copy(propSrc, pkgDict['propPath'])
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response = testController.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response.status_int == 200, 'Manifest Post assert'
LOG.debug('response body = %s' % response.body)
body = json.loads(response.body)
# wait until the status is done
tm = time.time()
now = tm
while (tm + 120 > now):
response = testController.app.get(body['status'])
body = json.loads(response.body)
LOG.debug("createManifest ********** progress = %s" % body['progress'])
if (int(body['progress']) == 100):
break
time.sleep(0.1)
now = time.time()
assert tm + 120 >= now, 'Create manifest timed out'
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
LOG.debug ('Status response body = %s' % str(body))
assert int(body['progress']) == 100
def test_same_pkg_download_parallel(self):
packages = ['http://www.stackscaling.com/agentrepo/pkgA-1.2.0.unix.cronus']
manifest1 = 'bar'
manifest2 = 'blah'
service1 = 'foo'
service2 = 'lah'
try:
for pkg in packages:
mockDownloadPkg(pkg)
path = ServiceController.manifestPath(service1)
os.makedirs(path)
path = ServiceController.installedPkgPath(service1)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service1)
os.makedirs(path)
path = ServiceController.manifestPath(service2)
os.makedirs(path)
path = ServiceController.installedPkgPath(service2)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service2)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response1.status_int == 200, 'Manifest Post assert'
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response2.status_int == 200, 'Manifest Post assert'
checkStatus(self, 'create manifest bar', response1, timeout = 25)
checkStatus(self, 'create manifest baz', response2, timeout = 25)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'pkgA-1.2.0.unix.cronus')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'pkgA')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('lah', 'blah'), 'pkgA')))
def test_same_pkg_download_parallel(self):
packages = ['http://repository.qa.ebay.com/cronus/test-data/agent/perlserver-1.0.0.unix.cronus']
manifest1 = 'bar'
manifest2 = 'blah'
service1 = 'foo'
service2 = 'lah'
try:
path = ServiceController.manifestPath(service1)
os.makedirs(path)
path = ServiceController.installedPkgPath(service1)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service1)
os.makedirs(path)
path = ServiceController.manifestPath(service2)
os.makedirs(path)
path = ServiceController.installedPkgPath(service2)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service2)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response1 = self.app.post(url(controller = 'manifest', action = 'post', service = service1, manifest = manifest1),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response1.status_int == 200, 'Manifest Post assert'
response2 = self.app.post(url(controller = 'manifest', action = 'post', service = service2, manifest = manifest2),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response2.status_int == 200, 'Manifest Post assert'
checkStatus(self, 'create manifest bar', response1, timeout = 25)
checkStatus(self, 'create manifest baz', response2, timeout = 25)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('lah', 'blah'), 'perlserver')))
def test_post3_pkg_already_installed(self):
createManifest(self)
# now lets remove the manifest path
path = os.path.join(manifestutil.manifestPath('foo', 'bar'))
shutil.rmtree(path)
# now create the manifest again
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
def test_post3_already_installed_manifest(self):
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']})
response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertEquals(201, response.status_int, 'Manifest Post assert')
body = json.loads(response.body)
assert response.status_int == 201, 'Manifest Post assert'
def test_post3_already_installed_manifest(self):
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
body = json.dumps({'package' : ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus']})
response = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "bar"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertEquals(201, response.status_int, 'Manifest Post assert')
body = json.loads(response.body)
assert response.status_int == 201, 'Manifest Post assert'
def test_get_installed_pkgs_filter_inprogress(self):
from agent.lib.package import PackageUtil
serviceName = '.sbe.appService.SI1'
manifestName = 'manifestA'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf'
installed_pkg_path = ServiceController.installedPkgPath(serviceName)
installed_packages = PackageUtil.getAllInstalledPackages(installed_pkg_path)
self.assertEquals(2, len(installed_packages))
for path in installed_packages:
os.mkdir(path + '.inprogress')
installed_packages = PackageUtil.getAllInstalledPackages(installed_pkg_path)
self.assertEquals(2, len(installed_packages))
def test_get_installed_pkgs_filter_inprogress(self):
from agent.lib.package import PackageUtil
serviceName = '.sbe.appService.SI1'
manifestName = 'manifestA'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf'
installed_pkg_path = ServiceController.installedPkgPath(serviceName)
installed_packages = PackageUtil.getAllInstalledPackages(installed_pkg_path)
self.assertEquals(2, len(installed_packages))
for path in installed_packages:
os.mkdir(path + '.inprogress')
installed_packages = PackageUtil.getAllInstalledPackages(installed_pkg_path)
self.assertEquals(2, len(installed_packages))
def test_package_reuse(self):
createManifest(self)
body = json.dumps({'package' : ['/packages/perlserver']})
body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
for _ in range(10):
if islink(os.path.join(manifestutil.manifestPath('foo', 'baz'), 'perlserver')):
break
time.sleep(1)
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'baz'), 'perlserver')))
def test_package_reuse(self):
createManifest(self)
body = json.dumps({'package' : ['/packages/perlserver']})
body = self.app.post(url(controller = 'manifest', action = 'post', service = "foo", manifest = "baz"),
headers = {'Content-Type' : 'application/json'},
params = body, expect_errors = True)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
for _ in range(10):
if islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')):
break
time.sleep(1)
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'baz'), 'perlserver')))
def test_inprogress_pkg_download(self):
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress')
inprogressFile = open(inprogressPath, 'w')
inprogressFile.write('somegarbage')
inprogressFile.close()
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(manifestutil.manifestPath('foo', 'bar'), 'perlserver')))
def test_inprogress_pkg_download(self):
service = 'foo'
try:
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
inprogressPath = os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus.inprogress')
inprogressFile = open(inprogressPath, 'w')
inprogressFile.write('somegarbage')
inprogressFile.close()
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def test_post3_pkg_already_installed(self):
createManifest(self)
# now lets remove the manifest path
path = os.path.join(ManifestController.manifestPath('foo', 'bar'))
if os.name == 'nt':
cmd = 'rm -r %s' % path
LOG.debug("running command %s" % cmd)
os.system(cmd)
else:
shutil.rmtree(path)
# now create the manifest again
createManifest(self)
self.assertTrue(os.path.exists(os.path.join(PackageMgr.packagePath(), 'perlserver-1.0.0.unix.cronus')))
self.assertTrue(os.path.exists(os.path.join(ServiceController.installedPkgPath('foo'), 'perlserver', '1.0.0.unix',
'cronus', 'scripts', 'activate')))
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'bar'), 'perlserver')))
def doRun(self):
""" Main body of the thread
Progress Info:
Look at the list of packages this manifest has and figure out the progress of this manifest
Progress 1 = manifest directory created
Progress 2-80 = all packages downloaded
Progress 81-99 = all packages untarred
Progress 100 = all links done
"""
inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(self.__service, self.__manifest))
try:
if self.__service != 'agent':
utils.checkDiskFull()
installedPkgPath = ServiceController.installedPkgPath(self.__service)
# This shouldn'trn happen but make sure there isn'trn already a manifest directory in progress
if (os.path.isdir(inProgressPath)):
# probably another manifest create thread died out half way.
# Cleanup and reattempt manifest creation
LOG.debug('Manifest inprogress found for service/manifest (%s/%s). Will cleanup and retry' % (self.__service, self.__manifest))
shutil.rmtree(inProgressPath)
# make sure that the service path and installed package path exists
if (not os.path.isdir(ServiceController.manifestPath(self.__service)) or
not os.path.isdir(ServiceController.installedPkgPath(self.__service))):
errCode = Errors.SERVICE_NOT_FOUND
msg = 'Service (%s) for manifest (%s) not found' % (self.__service, self.__manifest)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
return
# ok create the manifest path
os.mkdir(inProgressPath)
self._updateProgress(1)
# figure out which of the packages are already there
remainingPackages = {}
for pkgUri in self.__packages:
pkgDict = PackageUtil.parseUri(pkgUri)
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
pkgName = pkgDict['packageName']
if (not os.path.exists(pkgPath)) or ((self.__forcePackages is not None) and pkgName in self.__forcePackages):
remainingPackages[pkgUri] = pkgDict
else:
symlink(pkgPath, os.path.join(inProgressPath, pkgDict['packageName']))
if self.__attemptDownload:
# now make sure all the packages are downloaded
try:
self._downloadPackages(remainingPackages.keys(), skipProp = self.__skipProp)
except AgentException as exc:
# check if it is download error, then modify exception appropriately
if exc.getCode() == Errors.DC_FAILED_DOWNLOAD:
exc = AgentException(Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED,
'Manifest (%s/%s) failed downloading package - %s'
% (self.__service, self.__manifest, exc.getMsg()))
raise exc
else:
if len(remainingPackages) > 0:
raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
'Create Manifest (%s/%s) failed since package is not present and download has been disabled'
% (self.__service, self.__manifest))
LOG.info('Completed download all packages for (%s/%s)' % (self.__service, self.__manifest))
# now untar the packages
import re
pkgSuffix = '.%s' % re.sub(r"\W", "", self.__manifest)
self._untarPackages(remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service), 0, self.__forcePackages, pkgSuffix)
LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__manifest))
# now create the links
for pkgDict in remainingPackages.itervalues():
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
linkPath = os.path.join(inProgressPath, pkgDict['packageName'])
# validate target folder does exist
if not os.path.exists(pkgPath):
raise AgentException(Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath)
symlink(pkgPath, linkPath)
# now move the inProgressPath to the final path
manifestContentPath = ManifestController.manifestContentPath(self.__service, self.__manifest)
os.rename(inProgressPath, ManifestController.manifestPath(self.__service, self.__manifest))
mfContentsFile = file(manifestContentPath, 'w')
for pkgUri in self.__packages:
mfContentsFile.write(('%s%s') % (pkgUri, os.linesep))
mfContentsFile.close()
LOG.info('Completed create manifest directories for (%s/%s)' % (self.__service, self.__manifest))
LOG.info('Completed create manifest for (%s/%s)' % (self.__service, self.__manifest))
self._updateStatus(progress = 100)
except AgentException as exc:
LOG.info(exc.getMsg())
self._updateStatus(httpStatus = 500, error = exc.getCode(), errorMsg = exc.getMsg())
except Exception as exc:
errCode = Errors.UNKNOWN_ERROR
msg = 'Unknown error for (%s/%s) - %s - %s' % (self.__service, self.__manifest,
str(exc), traceback.format_exc(2))
LOG.info(msg)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
finally:
# clean up intermediate progress
try:
shutil.rmtree(inProgressPath)
except OSError:
pass
def doRun(self):
""" Main body of the thread
Progress Info:
Look at the list of packages this manifest has and figure out the progress of this manifest
Progress 1 = manifest directory created
Progress 2-80 = all packages downloaded
Progress 81-99 = all packages untarred
Progress 100 = all links done
"""
try:
if self.__service != 'agent':
utils.checkDiskFull()
installedPkgPath = manifestutil.installedPkgRootPath(self.__service)
# figure out which of the packages are already there
remainingPackages = {}
pkgDict = PackageUtil.parseUri(self.__package)
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
remainingPackages[self.__package] = pkgDict
if (not os.path.exists(pkgPath)):
# now make sure all the packages are downloaded
try:
self._downloadPackages([self.__package])
except AgentException as exc:
# check if it is download error, then modify exception appropriately
if exc.getCode() == Errors.DC_FAILED_DOWNLOAD:
exc = AgentException(Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED,
'Manifest (%s/%s) failed downloading package - %s'
% (self.__service, self.__module, exc.getMsg()))
raise exc
LOG.info('Completed download packages for (%s/%s)' % (self.__service, self.__module))
# now untar the packages
self._untarPackages(remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service))
LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__module))
# now create the links
for pkgDict in remainingPackages.itervalues():
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
modulePath = os.path.join(manifestutil.moduleRootPath(self.__service), self.__module)
# validate target folder does exist
if not os.path.exists(pkgPath):
raise AgentException(Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath)
# remove existing module
isExistingModule = os.path.exists(modulePath)
if isExistingModule:
execThread = self._getBuiltThread('deactivate')
if execThread:
super(ModuleCreate, self)._runExeThread(execThread)
rmlink(modulePath)
execThread = self._getBuiltThread('activate')
if execThread:
super(ModuleCreate, self)._runExeThread(execThread)
symlink(pkgPath, modulePath)
# load controllers from package
manifestutil.processModule(self.__service, self.__module, not isExistingModule)
LOG.info('Completed create module for (%s/%s)' % (self.__service, self.__module))
self._updateStatus(progress = 100)
except AgentException as exc:
LOG.info(exc.getMsg())
self._updateStatus(httpStatus = 500, error = exc.getCode(), errorMsg = exc.getMsg())
except Exception as exc:
errCode = Errors.UNKNOWN_ERROR
msg = 'Unknown error for (%s/%s) - %s - %s' % (self.__service, self.__module,
str(exc), traceback.format_exc(2))
LOG.info(msg)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
def doRun(self):
""" Main body of the thread
Progress Info:
Look at the list of packages this manifest has and figure out the progress of this manifest
Progress 1 = manifest directory created
Progress 2-80 = all packages downloaded
Progress 81-99 = all packages untarred
Progress 100 = all links done
"""
try:
if self.__service != 'agent':
utils.checkDiskFull()
installedPkgPath = manifestutil.installedPkgRootPath(
self.__service)
# figure out which of the packages are already there
remainingPackages = {}
pkgDict = PackageUtil.parseUri(self.__package)
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'],
pkgDict['packageVersion'])
remainingPackages[self.__package] = pkgDict
if (not os.path.exists(pkgPath)):
# now make sure all the packages are downloaded
try:
self._downloadPackages([self.__package])
except AgentException as exc:
# check if it is download error, then modify exception appropriately
if exc.getCode() == Errors.DC_FAILED_DOWNLOAD:
exc = AgentException(
Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED,
'Manifest (%s/%s) failed downloading package - %s'
% (self.__service, self.__module, exc.getMsg()))
raise exc
LOG.info('Completed download packages for (%s/%s)' %
(self.__service, self.__module))
# now untar the packages
self._untarPackages(
remainingPackages, self.__service,
ServiceController.installedPkgPath(self.__service))
LOG.info('Completed untar all packages for (%s/%s)' %
(self.__service, self.__module))
# now create the links
for pkgDict in remainingPackages.itervalues():
pkgPath = os.path.join(installedPkgPath,
pkgDict['packageName'],
pkgDict['packageVersion'])
modulePath = os.path.join(
manifestutil.moduleRootPath(self.__service), self.__module)
# validate target folder does exist
if not os.path.exists(pkgPath):
raise AgentException(
Errors.PACKAGE_PATH_ERROR,
'invalid untarred package at %s' % pkgPath)
# remove existing module
isExistingModule = os.path.exists(modulePath)
if isExistingModule:
execThreadTuple = self._getBuiltThread('deactivate')
super(ModuleCreate,
self)._runExeThread(self, execThreadTuple)
rmlink(modulePath)
execThreadTuple = self._getBuiltThread('activate')
super(ModuleCreate, self)._runExeThread(execThreadTuple)
symlink(pkgPath, modulePath)
# load controllers from package
manifestutil.processModule(self.__service, self.__module,
not isExistingModule)
LOG.info('Completed create module for (%s/%s)' %
(self.__service, self.__module))
self._updateStatus(progress=100)
except AgentException as exc:
LOG.info(exc.getMsg())
self._updateStatus(httpStatus=500,
error=exc.getCode(),
errorMsg=exc.getMsg())
except Exception as exc:
errCode = Errors.UNKNOWN_ERROR
msg = 'Unknown error for (%s/%s) - %s - %s' % (
self.__service, self.__manifest, str(exc),
traceback.format_exc(2))
LOG.info(msg)
self._updateStatus(httpStatus=500, error=errCode, errorMsg=msg)
def createManifest(
testController,
packages=[
'http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus'
],
manifest='bar',
service='foo',
createDirs=True):
try:
path = ServiceController.servicePath(service)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'modules'))
os.makedirs(os.path.join(path, '.appdata'))
os.makedirs(os.path.join(path, '.data'))
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
try:
for pkgUri in packages:
pkgDict = PackageUtil.parseUri(pkgUri)
pkgName = pkgDict['package']
localPkgRoot = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'packages')
src = os.path.join(localPkgRoot, pkgName)
shutil.copy(src, pkgDict['packagePath'])
propPkgName = pkgDict['propName']
propSrc = os.path.join(localPkgRoot, propPkgName)
shutil.copy(propSrc, pkgDict['propPath'])
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package': packages})
response = testController.app.post(
url(controller='manifest',
action='post',
service=service,
manifest=manifest),
headers={'Content-Type': 'application/json'},
params=body)
assert response.status_int == 200, 'Manifest Post assert'
LOG.debug('response body = %s' % response.body)
body = json.loads(response.body)
# wait until the status is done
tm = time.time()
now = tm
while (tm + 120 > now):
response = testController.app.get(body['status'])
body = json.loads(response.body)
LOG.debug("createManifest ********** progress = %s" %
body['progress'])
if (int(body['progress']) == 100):
break
time.sleep(0.1)
now = time.time()
assert tm + 120 >= now, 'Create manifest timed out'
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
LOG.debug('Status response body = %s' % str(body))
assert int(body['progress']) == 100