def test_parsing_bad_extension(self):
# test bad extension
try:
PackageUtil.parseUri('http://foo.com/repo/agent-0.1.1a.cronus2', PackageMgr.packagePath())
assert False, 'bad extension'
except AgentException:
pass
def test_parsing_bad_name(self):
# test bad name
try:
PackageUtil.parseUri('http://foo.com/repo/a?gent-0.1.1.unix.cronus', PackageMgr.packagePath())
assert False, 'bad name'
except AgentException:
pass
def test_parsing_garbage(self):
# test the url parsing
try:
PackageUtil.parseUri('asdfawert892375[jrgjhnjbuy43w5897t^!R$#W_R(@$^(_#$(', PackageMgr.packagePath())
assert False, 'parse garbage'
except AgentException:
pass
def test_parsing_bad_extension(self):
# test bad extension
try:
PackageUtil.parseUri('http://foo.com/repo/agent-0.1.1a.cronus2',
PackageMgr.packagePath())
assert False, 'bad extension'
except AgentException:
pass
def test_cleanupOrphanedPackages2(self):
from agent.lib.utils import rchown
from agent.controllers.service import ServiceController
from agent.controllers.manifest import ManifestController
from agent.tests.unit.test_util import createManifest
if os.name == 'nt':
pass
else:
serviceName = 'service2'
manifestName = 'manifestA'
serviceName1 = 'service3'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"
],
service=serviceName1,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
installedPkgPath = os.path.realpath(
os.path.join(ServiceController.serviceRootPath(), serviceName,
'installed-packages'))
installedPkgPath1 = os.path.realpath(
os.path.join(ServiceController.serviceRootPath(), serviceName1,
'installed-packages'))
if (not os.name == 'nt'):
import pwd
uname = pylons.config['agent_user_account']
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
rchown(installedPkgPath, uid, gid)
rchown(installedPkgPath1, uid, gid)
self.assertEquals(len(os.listdir(installedPkgPath)), 1)
self.assertEquals(len(os.listdir(installedPkgPath1)), 1)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName1,
manifest=manifestName),
expect_errors=True)
pylons.config['packageMgr_install_package_age'] = 0
pylons.config['packageMgr_install_package_min_age'] = 0.0
PackageUtil.cleanupOrphanedPackages()
self.assertEquals(len(os.listdir(installedPkgPath)), 0)
self.assertEquals(len(os.listdir(installedPkgPath1)), 0)
def test_parsing_bad_name(self):
# test bad name
try:
PackageUtil.parseUri(
'http://foo.com/repo/a?gent-0.1.1.unix.cronus',
PackageMgr.packagePath())
assert False, 'bad name'
except AgentException:
pass
def test_parsing_garbage(self):
# test the url parsing
try:
PackageUtil.parseUri(
'asdfawert892375[jrgjhnjbuy43w5897t^!R$#W_R(@$^(_#$(',
PackageMgr.packagePath())
assert False, 'parse garbage'
except AgentException:
pass
def getPackageDownloadThread(self, packageUri, skipProp = True):
"""
getPackage(package)
@params package - URI (e.g. http://repo/repo/package-0.1.4.cronus)
in case of bittorrent - http://repo/repo/package-0.1.4.cronus.torrent
get the DownloadThread object for a particular package URI
If a DownloadThread does not exist, create one
"""
# try to get create the package
uriDict = PackageUtil.parseUri(packageUri)
with self.__inProgressLock:
# if the package is already in progress, just return that object
if (uriDict['package'] in self.__inProgressPackages):
thread = self.__inProgressPackages[uriDict['package']]
if thread.isAlive():
return thread
# start package download - either it is a new download, or a prev thread errored-out
appGlobal = pylons.config['pylons.app_globals']
downloadThread = DownloadThread(appGlobal.threadMgr, packageUri, packageloc=None, skipProp = skipProp)
downloadThread.start()
# add to the list
self.__inProgressPackages[uriDict['package']] = downloadThread
return downloadThread
def getPackageDownloadThread(self, packageUri, skipProp=True):
"""
getPackage(package)
@params package - URI (e.g. http://repo/repo/package-0.1.4.cronus)
in case of bittorrent - http://repo/repo/package-0.1.4.cronus.torrent
get the DownloadThread object for a particular package URI
If a DownloadThread does not exist, create one
"""
# try to get create the package
uriDict = PackageUtil.parseUri(packageUri)
with self.__inProgressLock:
# if the package is already in progress, just return that object
if (uriDict['package'] in self.__inProgressPackages):
thread = self.__inProgressPackages[uriDict['package']]
if thread.isAlive():
return thread
# start package download - either it is a new download, or a prev thread errored-out
appGlobal = pylons.config['pylons.app_globals']
downloadThread = DownloadThread(appGlobal.threadMgr,
packageUri,
packageloc=None,
skipProp=skipProp)
downloadThread.start()
# add to the list
self.__inProgressPackages[uriDict['package']] = downloadThread
return downloadThread
def mockDownloadPkg(pkgUri):
""" mock download a cronus package """
pkgDict = PackageUtil.parseUri(pkgUri)
pkgName = pkgDict['package']
localPkgRoot = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'packages')
src = os.path.join(localPkgRoot, pkgName)
shutil.copy(src, pkgDict['packagePath'])
def test_parsing_good_url(self):
# test good url
uri = 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus?query=blah#foo'
uriDict = PackageUtil.parseUri(uri, PackageMgr.packagePath())
assert uriDict['uri'] == uri
assert uriDict['scheme'] == 'http'
assert uriDict['uripath'] == '/repo/agent-0.1.2.unix.cronus'
assert uriDict['package'] == 'agent-0.1.2.unix.cronus'
assert uriDict['packageNameVersion'] == 'agent-0.1.2.unix'
assert uriDict['packageName'] == 'agent'
assert uriDict['packageVersion'] == '0.1.2.unix'
assert uriDict['packageVersionMajor'] == '0'
assert uriDict['packageVersionMinor'] == '1'
assert uriDict['packageVersionBuild'] == '2'
assert uriDict['packagePlat'] == 'unix'
assert uriDict['propName'] == 'agent-0.1.2.unix.cronus.prop'
assert uriDict[
'propUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.prop?query=blah#foo'
assert uriDict['packagePath'] == os.path.join(
PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus')
assert uriDict['inProgressPackagePath'] == os.path.join(
PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.inprogress')
assert uriDict['propPath'] == os.path.join(
PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.prop')
assert uriDict[
'torrentUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.torrent?query=blah#foo'
assert uriDict['torrentPath'] == os.path.join(
PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.torrent')
assert uriDict['torrentName'] == 'agent-0.1.2.unix.cronus.torrent'
def deletePackage(self, package):
""" secret !!hack!! API to delete one cronus package in the packages folder. """
try:
LOG.info('secret delete called for package %s' % package)
packagesPath = PackageMgr.packagePath()
thisPkgPath = os.path.join(packagesPath, package + '.cronus')
thisPropPath = os.path.join(packagesPath, package + '.cronus.prop')
thisInprogressPath = PackageUtil.inProgressPath(thisPkgPath)
PackageUtil.cleanUpPackage(thisInprogressPath, thisPkgPath, thisPropPath)
return doneResult(request, response, controller=self)
except Exception as excp:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when deleting package %s, %s - %s' %
(package, str(excp), traceback.format_exc(2)),
controller = self)
def startdownload(self):
"""
Download using http. The protocol is chosen based on package uri.
Target folder is packageloc relative to repo_root.
"""
try:
utils.checkDiskFull()
reqjson = json.loads(request.body)
package = reqjson['package']
packageloc = str(reqjson['packageloc'])
LOG.info('Request received for StartDownload %s' % packageloc)
appGlobal = config['pylons.app_globals']
LOG.info('Starting a new StartDownload Thread %s' % package)
if type(package) == list:
downloadThread = PackageDownload(appGlobal.threadMgr, package)
else:
cat = PackageUtil.getPackageKey(package)
downloadThread = DownloadThread(appGlobal.threadMgr, package, packageloc, category = [cat])
self.injectJobCtx(downloadThread)
downloadThread.start()
downloadThread.threadMgrEvent.wait()
return statusResult(request, response, downloadThread, controller = self)
except AgentException as excep:
return errorResult(request, response, error = excep.getCode(), errorMsg = excep.getMsg(), controller = self)
except Exception as excp:
errorMsg = 'Exception downloading %s - traceback %s' % (str(excp), traceback.format_exc(2))
return errorResult(request, response, error = Errors.UNKNOWN_ERROR, errorMsg = errorMsg, controller = self)
def __download_prop_file(self):
""" download prop file and validate """
# retry 3 times download prop file
for _ in range(3):
try:
sotimeout = float(pylons.config['download_thread_sotimeout'])
proxies = json.loads(pylons.config['urlgrabber_proxies'])
urlgrabber.urlgrab(self.__uriDict['propUri'],
self.__uriDict['propPath'],
keepalive=0,
timeout=sotimeout,
proxies=proxies)
break
except Exception:
randsleep = randint(30, 60)
time.sleep(randsleep)
if (not os.path.exists(self.__uriDict['propPath'])):
raise AgentException(
Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) does not exist' % (self.__uriDict['propPath']))
if not PackageUtil.validateProp(self.__uriDict['propPath']):
raise AgentException(
Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) failed validation' %
(self.__uriDict['propPath']))
def __init__(self,
threadMgr,
packageUri,
packageloc,
path=None,
category=None,
skipProp=True):
# cat = 'DC_DowloadThread' + str(datetime.now())
AgentThread.__init__(self,
threadMgr,
cat=category,
name='download_thread')
self.__path = pylons.config['repo_root']
if path is not None:
self.__path = path
# check to see if the package path exists
if (not os.path.isdir(self.__path)):
msg = 'Package path(%s) does not exist' % self.__path
LOG.error(msg)
raise AgentException(Errors.PACKAGE_PATH_ERROR, msg)
self.__uriDict = PackageUtil.parseUri(packageUri, self.__path,
packageloc)
self.__prop = {}
self.__error = None
self.__progress = 0.0
self.__timeouts = None
self.__skipProp = skipProp
def __download_prop_file(self):
""" download prop file and validate """
# retry 3 times download prop file
for _ in range(3):
try:
sotimeout = float(pylons.config['download_thread_sotimeout'])
proxies = json.loads(pylons.config['urlgrabber_proxies'])
urlgrabber.urlgrab(
self.__uriDict['propUri'],
self.__uriDict['propPath'],
keepalive = 0,
timeout = sotimeout,
proxies = proxies)
break
except Exception:
randsleep = randint(30, 60)
time.sleep(randsleep)
if (not os.path.exists(self.__uriDict['propPath'])):
raise AgentException(Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) does not exist' % (self.__uriDict['propPath']))
if not PackageUtil.validateProp(self.__uriDict['propPath']):
raise AgentException(Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) failed validation' % (self.__uriDict['propPath']))
def deletePackage(self, package):
""" secret !!hack!! API to delete one cronus package in the packages folder. """
try:
LOG.info('secret delete called for package %s' % package)
packagesPath = PackageMgr.packagePath()
thisPkgPath = os.path.join(packagesPath, package + '.cronus')
thisPropPath = os.path.join(packagesPath, package + '.cronus.prop')
thisInprogressPath = PackageUtil.inProgressPath(thisPkgPath)
PackageUtil.cleanUpPackage(thisInprogressPath, thisPkgPath, thisPropPath)
return doneResult(request, response, controller=self)
except Exception as excp:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error when deleting package %s, %s - %s' %
(package, str(excp), traceback.format_exc(2)),
controller = self)
def test_get_installed_pkgs_filter_inprogress(self):
from agent.lib.package import PackageUtil
serviceName = '.sbe.appService.SI1'
manifestName = 'manifestA'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf'
installed_pkg_path = ServiceController.installedPkgPath(serviceName)
installed_packages = PackageUtil.getAllInstalledPackages(installed_pkg_path)
self.assertEquals(2, len(installed_packages))
for path in installed_packages:
os.mkdir(path + '.inprogress')
installed_packages = PackageUtil.getAllInstalledPackages(installed_pkg_path)
self.assertEquals(2, len(installed_packages))
def test_get_installed_pkgs_filter_inprogress(self):
from agent.lib.package import PackageUtil
serviceName = '.sbe.appService.SI1'
manifestName = 'manifestA'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf'
installed_pkg_path = ServiceController.installedPkgPath(serviceName)
installed_packages = PackageUtil.getAllInstalledPackages(installed_pkg_path)
self.assertEquals(2, len(installed_packages))
for path in installed_packages:
os.mkdir(path + '.inprogress')
installed_packages = PackageUtil.getAllInstalledPackages(installed_pkg_path)
self.assertEquals(2, len(installed_packages))
def createManifest(testController,
packages = ['http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus'],
manifest = 'bar', service = 'foo', createDirs = True):
try:
path = ServiceController.servicePath(service)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'modules'))
os.makedirs(os.path.join(path, '.appdata'))
os.makedirs(os.path.join(path, '.data'))
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
try:
for pkgUri in packages:
pkgDict = PackageUtil.parseUri(pkgUri)
pkgName = pkgDict['package']
localPkgRoot = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'packages')
src = os.path.join(localPkgRoot, pkgName)
shutil.copy(src, pkgDict['packagePath'])
propPkgName = pkgDict['propName']
propSrc = os.path.join(localPkgRoot, propPkgName)
shutil.copy(propSrc, pkgDict['propPath'])
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package' : packages})
response = testController.app.post(url(controller = 'manifest', action = 'post', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response.status_int == 200, 'Manifest Post assert'
LOG.debug('response body = %s' % response.body)
body = json.loads(response.body)
# wait until the status is done
tm = time.time()
now = tm
while (tm + 120 > now):
response = testController.app.get(body['status'])
body = json.loads(response.body)
LOG.debug("createManifest ********** progress = %s" % body['progress'])
if (int(body['progress']) == 100):
break
time.sleep(0.1)
now = time.time()
assert tm + 120 >= now, 'Create manifest timed out'
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
LOG.debug ('Status response body = %s' % str(body))
assert int(body['progress']) == 100
def test_cleanUpAllInstalledPackages(self):
from agent.lib.utils import rchown
from agent.tests.unit.test_util import createManifest
#import pdb;pdb.set_trace();
serviceName = 'service3'
manifestName = 'manifestB'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
installedPkgPath = os.path.realpath(os.path.join(manifestutil.serviceRootPath() , serviceName, 'installed-packages'))
uname = configutil.getAgentUser()
uid, gid = utils.getUidGid(uname)
rchown(installedPkgPath, uid, gid)
pkgs = []
for pkg in os.listdir(installedPkgPath):
pkgs.append(os.path.join(installedPkgPath, pkg))
pkgVers = []
for pkg in pkgs:
for pkgVer in os.listdir(pkg):
pkgVers.append(os.path.join(pkg, pkgVer))
self.assertEquals(len(os.listdir(installedPkgPath)) , 1)
self.app.delete(url(controller = 'manifest', action = 'delete', service = serviceName, manifest = manifestName), expect_errors = True)
#import pdb; pdb.set_trace();
symLinks = [] #ManifestController.getAllSymLinks(serviceName)
orphans = set(pkgVers) - set(symLinks)
age = pylons.config['packageMgr_install_package_age']
minage = pylons.config['packageMgr_install_package_min_age']
pylons.config['packageMgr_install_package_age'] = 0
pylons.config['packageMgr_install_package_min_age'] = 0.0
self.assertEquals(len(os.listdir(installedPkgPath)) , 1)
PackageUtil.cleanupInstalledPkgs(installedPkgPath, orphans)
#PackageUtil.cleanUpInstalledPkgs(installedPkgPath, orphans)
self.assertEquals(len(os.listdir(installedPkgPath)) , 0)
pylons.config['packageMgr_install_package_age'] = age
pylons.config['packageMgr_install_package_min_age'] = minage
def test_cleanupOrphanedPackages(self):
from agent.lib.utils import rchown
from agent.controllers.service import ServiceController
from agent.controllers.manifest import ManifestController
from agent.tests.unit.test_util import createManifest
if os.name == 'nt':
pass
else:
serviceName = 'service2'
manifestName = 'manifestA'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
installedPkgPath = os.path.realpath(
os.path.join(ServiceController.serviceRootPath(), serviceName,
'installed-packages'))
if (not os.name == 'nt'):
import pwd
uname = pylons.config['agent_user_account']
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
rchown(installedPkgPath, uid, gid)
#installedPkgPath = os.path.realpath(os.path.join('/ebay/srengarajan/e2e/local-deploy/service_nodes/' , '.sbe.appService.SI1', 'installed-packages' ))
#pkgs = os.listdir('C:\\srini\\projects\\gc\\agent\\trunk')
self.assertEquals(len(os.listdir(installedPkgPath)), 1)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
pylons.config['packageMgr_install_package_age'] = 0
pylons.config['packageMgr_install_package_min_age'] = 0.0
time.sleep(1)
PackageUtil.cleanupOrphanedPackages()
self.assertEquals(len(os.listdir(installedPkgPath)), 0)
def _findPackagesToDownload(self, packages, packageRootPath):
''' check which packages exist and which need to be downloaded '''
# figure out which of the packages are already there
remainingPackages = {}
for one_package in packages:
one_package_dict = PackageUtil.parseUri(one_package)
one_package_path = os.path.join(packageRootPath, one_package_dict['packageName'], one_package_dict['packageVersion'])
if (not os.path.exists(one_package_path)):
remainingPackages[one_package] = one_package_dict
return remainingPackages
def _findPackagesToDownload(self, packages, packageRootPath):
''' check which packages exist and which need to be downloaded '''
# figure out which of the packages are already there
remainingPackages = {}
for one_package in packages:
one_package_dict = PackageUtil.parseUri(one_package)
one_package_path = os.path.join(packageRootPath, one_package_dict['packageName'], one_package_dict['packageVersion'])
if (not os.path.exists(one_package_path)):
remainingPackages[one_package] = one_package_dict
return remainingPackages
def test_forceDeletePackages(self):
#import pdb;pdb.set_trace();
from agent.tests.unit.test_util import createManifest
if os.name == 'posix':
serviceName = 'service4'
manifestName = 'manifestb'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
age = pylons.config['packageMgr_package_age']
pylons.config['packageMgr_package_age'] = 86400
minage = pylons.config['packageMgr_package_min_age']
pylons.config['packageMgr_package_min_age'] = 0.0
#appGlobal = pylons.config['pylons.app_globals']
#appGlobal.packageMgr.start()
disk_threshold = pylons.config[
'health_disk_usage_percent_threshold']
pylons.config['health_disk_usage_percent_threshold'] = 1
pylons.config['health_disk_usage_gc_threshold'] = 1
pkgs = PackageMgr.packagePath()
self.assertEquals(len(os.listdir(pkgs)), 4)
PackageUtil.forceCleanUpDownloadedPkgs()
self.assertEquals(len(os.listdir(pkgs)), 0)
pylons.config['packageMgr_package_age'] = age
pylons.config['packageMgr_package_min_age'] = minage
pylons.config[
'health_disk_usage_percent_threshold'] = disk_threshold
def doRun(self):
""" Main body of the thread """
try:
path = pylons.config['repo_root']
packagePath = os.path.join(path, self.__package)
propPath = os.path.join(path, (self.__package + '.prop'))
if os.path.exists(packagePath):
if (os.path.exists(propPath)
and PackageUtil.validateProp(propPath)
and PackageUtil.validatePackage(packagePath)):
os.utime(packagePath, None)
os.utime(propPath, None)
self._updateStatus(
progress=100,
result={'msg': 'Package %s is valid' % packagePath})
else:
msg = 'Package %s failed validation ' % packagePath
LOG.warning(msg)
self._updateStatus(httpStatus=500,
progress=100,
error=Errors.PACKAGE_CHECKSUM_ERROR,
errorMsg=msg)
else:
self._updateStatus(httpStatus=500,
progress=100,
error=Errors.PACKAGE_NOT_FOUND,
errorMsg=('Package not found %s' %
packagePath))
except Exception as excp:
errorMsg = 'Exception downloading %s - traceback %s' % (
str(excp), traceback.format_exc(2))
self._updateStatus(httpStatus=500,
progress=100,
error=Errors.UNKNOWN_ERROR,
errorMsg=errorMsg)
def doRun(self):
""" Main body of the thread """
try:
path = pylons.config['repo_root']
packagePath = os.path.join(path, self.__package)
propPath = os.path.join(path, (self.__package + '.prop'))
if os.path.exists(packagePath):
if (os.path.exists(propPath) and
PackageUtil.validateProp(propPath) and
PackageUtil.validatePackage(packagePath)):
os.utime(packagePath, None)
os.utime(propPath, None)
self._updateStatus(progress = 100, result = {'msg': 'Package %s is valid' % packagePath})
else:
msg = 'Package %s failed validation ' % packagePath
LOG.warning(msg)
self._updateStatus(httpStatus = 500, progress = 100, error = Errors.PACKAGE_CHECKSUM_ERROR, errorMsg = msg)
else:
self._updateStatus(httpStatus = 500, progress = 100, error = Errors.PACKAGE_NOT_FOUND, errorMsg = ('Package not found %s' % packagePath))
except Exception as excp:
errorMsg = 'Exception downloading %s - traceback %s' % (str(excp), traceback.format_exc(2))
self._updateStatus(httpStatus = 500, progress = 100, error = Errors.UNKNOWN_ERROR, errorMsg = errorMsg)
def test_getAllInstalledPackages(self):
from agent.tests.unit.test_util import createManifest
serviceName = 'service1'
manifestName = 'manifestA'
try:
createManifest(self, packages = ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"],
service = serviceName, manifest = manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
time.sleep(2)
installedPkgPath = os.path.join(manifestutil.servicePath(serviceName), 'installed-packages')
pkgs = PackageUtil.getAllInstalledPackages(installedPkgPath)
self.assertEquals(pkgs.__len__(), 2)
def deployservice(self, service):
""" activate manifest, if already active then skip """
LOG.info('deploy service for service(%s) with body: %s', service, request.body)
manifest = None
try:
appGlobal = config['pylons.app_globals']
# parse the body
if (request.body == ""):
return errorResult(request, response, Errors.INVALID_REQUEST,
'No body found in post command',
controller = self)
requestjson = json.loads(request.body)
packages = requestjson['package']
if 'manifest' in requestjson:
manifest = requestjson['manifest']
else:
manifest = PackageUtil.getPackageVersion(packages[-1])
serviceutil.createServiceIfNeeded(service)
# activate manifest if not already activated
if manifestutil.getActiveManifest(service) == manifest:
return doneResult(request, response, controller=self)
else:
# save metadata from payload
pushedData = {}
pushedData.update(requestjson)
for key in ['manifest', 'package']:
if key in pushedData:
del pushedData[key]
serviceutil.updateLcmMeta(service, pushedData)
# deploy
deployServiceThread = DeployService(appGlobal.threadMgr, service, manifest, packages)
self.injectJobCtx(deployServiceThread)
deployServiceThread.start()
deployServiceThread.threadMgrEvent.wait()
return statusResult(request, response, deployServiceThread, controller = self)
except Exception as excep:
msg = 'Unknown error for deployService(%s/%s) - %s - %s' % (service, manifest, str(excep), traceback.format_exc(2))
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = msg, controller = self)
def __init__(self, threadMgr, packageUri, packageloc, path = None, category = None, skipProp = True):
# cat = 'DC_DowloadThread' + str(datetime.now())
AgentThread.__init__(self, threadMgr, cat = category, name = 'download_thread')
self.__path = pylons.config['repo_root']
if path is not None:
self.__path = path
# check to see if the package path exists
if (not os.path.isdir(self.__path)):
msg = 'Package path(%s) does not exist' % self.__path
LOG.error(msg)
raise AgentException(Errors.PACKAGE_PATH_ERROR, msg)
self.__uriDict = PackageUtil.parseUri(packageUri, self.__path, packageloc)
self.__prop = {}
self.__error = None
self.__progress = 0.0
self.__timeouts = None
self.__skipProp = skipProp
def __init__(self, threadMgr, packageUri, packageloc, path = None, category = None, parentId = None):
AgentThread.__init__(self, threadMgr, cat = category, name = 'download_thread', parentId = parentId)
self._mergeOnFound = True
self.__path = pylons.config['repo_root']
if path is not None:
self.__path = path
# check to see if the package path exists
if (not os.path.isdir(self.__path)):
msg = 'Package path(%s) does not exist' % self.__path
LOG.error(msg)
raise AgentException(Errors.PACKAGE_PATH_ERROR, msg)
self.__uriDict = PackageUtil.parseUri(packageUri, self.__path, packageloc)
self.__prop = {}
self.__error = None
self.__progress = 0.0
self.__timeouts = None
self.__skipProp = configutil.getConfigAsBool('download_skip_prop')
def test_parsing_good_url(self):
# test good url
uri = 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus?query=blah#foo'
uriDict = PackageUtil.parseUri(uri, PackageMgr.packagePath())
assert uriDict['uri'] == uri
assert uriDict['scheme'] == 'http'
assert uriDict['uripath'] == '/repo/agent-0.1.2.unix.cronus'
assert uriDict['package'] == 'agent-0.1.2.unix.cronus'
assert uriDict['packageNameVersion'] == 'agent-0.1.2.unix'
assert uriDict['packageName'] == 'agent'
assert uriDict['packageVersion'] == '0.1.2.unix'
assert uriDict['packageVersionMajor'] == '0'
assert uriDict['packageVersionMinor'] == '1'
assert uriDict['packageVersionBuild'] == '2'
assert uriDict['packagePlat'] == 'unix'
assert uriDict['propName'] == 'agent-0.1.2.unix.cronus.prop'
assert uriDict['propUri'] == 'http://*****:*****@foo.com:8080/repo/agent-0.1.2.unix.cronus.prop?query=blah#foo'
assert uriDict['packagePath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus')
assert uriDict['inProgressPackagePath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.inprogress')
assert uriDict['propPath'] == os.path.join(PackageMgr.packagePath(), 'agent-0.1.2.unix.cronus.prop')
def getPackages(service, manifest):
""" return the list of packages in the manifest. The list is just the names not paths in the order specified during manifest creation.
@param service: name of the service
@param manifest: name of the manifest
@return: list with names of packages in it. If none, returns an empty list []
"""
mfContentPath = ManifestController.manifestContentPath(service, manifest)
if (os.path.exists(mfContentPath)):
packages = []
mfContentFile = file(mfContentPath, 'r')
for packageUri in mfContentFile:
pkgObj = PackageUtil.parseUri(packageUri.rstrip())
packages.append(pkgObj['packageName'])
return packages
else:
#Defaulting to list directory when manifest contents are not found.
path = ManifestController.manifestPath(service, manifest)
packages = os.listdir(path)
return sorted(packages)
def getPackages(service, manifest):
""" return the list of packages in the manifest. The list is just the names not paths in the order specified during manifest creation.
@param service: name of the service
@param manifest: name of the manifest
@return: list with names of packages in it. If none, returns an empty list []
"""
mfContentPath = ManifestController.manifestContentPath(
service, manifest)
if (os.path.exists(mfContentPath)):
packages = []
mfContentFile = file(mfContentPath, 'r')
for packageUri in mfContentFile:
pkgObj = PackageUtil.parseUri(packageUri.rstrip())
packages.append(pkgObj['packageName'])
return packages
else:
#Defaulting to list directory when manifest contents are not found.
path = ManifestController.manifestPath(service, manifest)
packages = os.listdir(path)
return sorted(packages)
def packagesInManifest(service, manifest=ACTIVE_MANIFEST):
""" return the list of packages in the manifest. The list is just the names not paths in the order specified during manifest creation.
@param __service: name of the __service
@param manifest: name of the manifest
@return: list with names of packages in it. If none, returns an empty list []
"""
#Defaulting to list directory when manifest contents are not found.
mfContentPath = manifestContentPath(service, manifest)
if (os.path.exists(mfContentPath)):
packages = []
mfContentFile = file(mfContentPath, 'r')
for packageUri in mfContentFile:
from agent.lib.package import PackageUtil
pkgObj = PackageUtil.parseUri(packageUri.rstrip())
packages.append(pkgObj['packageName'])
return packages
else:
path = manifestPath(service, manifest)
dirs = []
for fileName in os.listdir(path):
if os.path.isdir(os.path.join(path, fileName)):
dirs.append(fileName)
dirs.sort()
return dirs
def test_getAllInstalledPackages(self):
from agent.tests.unit.test_util import createManifest
from agent.controllers.service import ServiceController
serviceName = 'service1'
manifestName = 'manifestA'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus",
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgB-0.6.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
time.sleep(2)
installedPkgPath = os.path.join(
ServiceController.servicePath(serviceName), 'installed-packages')
pkgs = PackageUtil.getAllInstalledPackages(installedPkgPath)
self.assertEquals(pkgs.__len__(), 2)
def run(self):
"""
the garbage collection thread body
go through the list of packages and delete packages over a certain threshold
two variables affect operation
packageMgr_gc_freq = how log GC sleeps for between checks
pacageMgr_package_age = how old a package can get to before it's deleted
go through the list of inprogress threads and remove all threads that are done
"""
#pylint:disable=R0914, R0912, R0915
while (self.__stop == False):
time.sleep(self.__garbageFreq)
try:
# GC in progress hash
self.__inProgressLock.acquire()
try:
for key in self.__inProgressPackages.keys():
if (not self.__inProgressPackages[key].isAlive()):
del self.__inProgressPackages[key]
finally:
self.__inProgressLock.release()
if not self.__stop:
# go through all the packages and see if they are older than the threshold
for filename in glob.glob(os.path.join(PackageMgr.packagePath(), '*.cronus.inprogress')):
if (time.time() > os.path.getatime(filename) + float(pylons.config['packageMgr_package_age'])):
try:
GCLOG.info('Garbage collecting package(%s)' % filename)
LOG.info('Garbage collecting package(%s)' % filename)
os.remove(filename)
except OSError, osErr:
GCLOG.error('Unable to garbage collect %s - %s' % (filename, osErr))
LOG.error('Unable to garbage collect %s - %s' % (filename, osErr))
for filename in glob.glob(os.path.join(PackageMgr.packagePath(), '*.cronus')):
LOG.debug('garbage collection check in progress for filename %s' % filename)
try:
if (time.time() > os.path.getatime(filename) + float(pylons.config['packageMgr_package_age'])):
LOG.info('Garbage collecting package(%s)' % filename)
os.remove(filename)
if os.path.exists(filename + '.prop'):
os.remove(filename + '.prop')
if os.path.exists(filename + '.torrent'):
os.remove(filename + '.torrent')
if os.path.exists(filename + '.inprogress'):
os.remove(filename + '.inprogress')
except OSError, osErr:
GCLOG.error('Unable to garbage collect %s - %s' % (filename, osErr))
LOG.error('Unable to garbage collect %s - %s' % (filename, osErr))
#appGlobal = pylons.config['pylons.app_globals']
#import pdb;pdb.set_trace();
if not self.__stop:
PackageUtil.cleanupOrphanedPackages(False)
packageMount = pylons.config['agent_root']
if(not self.__stop and agenthealth.needAggressiveGC(packageMount)):
PackageUtil.forceCleanUpDownloadedPkgs()
# validate all services and remove rogue services
PackageMgr.cleanRogueService()
def test_cleanUpAllInstalledPackages(self):
from agent.lib.utils import rchown
from agent.controllers.service import ServiceController
from agent.controllers.manifest import ManifestController
from agent.tests.unit.test_util import createManifest
#import pdb;pdb.set_trace();
serviceName = 'service3'
manifestName = 'manifestB'
try:
createManifest(
self,
packages=[
"http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus"
],
service=serviceName,
manifest=manifestName)
except Exception as ex:
print 'exception thrown during mf %s' % str(ex)
installedPkgPath = os.path.realpath(
os.path.join(ServiceController.serviceRootPath(), serviceName,
'installed-packages'))
if (not os.name == 'nt'):
import pwd
uname = pylons.config['agent_user_account']
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
rchown(installedPkgPath, uid, gid)
#installedPkgPath = os.path.realpath(os.path.join('/ebay/srengarajan/e2e/local-deploy/service_nodes/' , '.sbe.appService.SI1', 'installed-packages' ))
#pkgs = os.listdir('C:\\srini\\projects\\gc\\agent\\trunk')
pkgs = []
for pkg in os.listdir(installedPkgPath):
pkgs.append(os.path.join(installedPkgPath, pkg))
pkgVers = []
for pkg in pkgs:
for pkgVer in os.listdir(pkg):
pkgVers.append(os.path.join(pkg, pkgVer))
self.assertEquals(len(os.listdir(installedPkgPath)), 1)
self.app.delete(url(controller='manifest',
action='delete',
service=serviceName,
manifest=manifestName),
expect_errors=True)
#import pdb; pdb.set_trace();
symLinks = [] #ManifestController.getAllSymLinks(serviceName)
orphans = set(pkgVers) - set(symLinks)
age = pylons.config['packageMgr_install_package_age']
minage = pylons.config['packageMgr_install_package_min_age']
pylons.config['packageMgr_install_package_age'] = 0
pylons.config['packageMgr_install_package_min_age'] = 0.0
self.assertEquals(len(os.listdir(installedPkgPath)), 1)
PackageUtil.cleanupInstalledPkgs(installedPkgPath, orphans)
#PackageUtil.cleanUpInstalledPkgs(installedPkgPath, orphans)
self.assertEquals(len(os.listdir(installedPkgPath)), 0)
pylons.config['packageMgr_install_package_age'] = age
pylons.config['packageMgr_install_package_min_age'] = minage
def _downloadPackages(self, packages, isFailedFatal = True):
"""
download all the packages
update progress
check for timeout
@params packages = list of package uri's
@throws AgentException
"""
try:
appGlobal = pylons.config['pylons.app_globals']
# globals all the remaining packages, download the packages
pkgObjs = []
for pkg in packages:
packageKey = PackageUtil.getPackageKey(pkg)
matchDlThreads = appGlobal.threadMgr.getThreadByCat(packageKey)
if matchDlThreads:
# found an inprogress download thread
dlThread = matchDlThreads[0]
else:
dlThread = DownloadThread(self._threadMgr, pkg, category=[packageKey], packageloc=None, parentId = self.getUuid())
contextutils.copyJobContexts(self, dlThread)
dlThread.start()
if (dlThread.getStatus().get('progress') != 100):
pkgObjs.append(dlThread)
self._checkStop()
# check that we have packages to download
if (len(pkgObjs) == 0):
return [] # nothing to download, so no failed packages
# now wait for all the packages to finish
liveThreadCount = len(pkgObjs)
timeoutNotSet = True
create_sleep_time = float(pylons.config['exec_thread_sleep_time']) * liveThreadCount
failed_packages = []
while (liveThreadCount > 0):
self._checkStop()
LOG.info('%s packages still downloading' % liveThreadCount)
time.sleep(create_sleep_time)
# go through all the packages and
# calculated average progress
# check that all packages are alive
totalProgress = 0
liveThreadCount = 0
timeouts = (0.0, 0.0)
for dlThread in pkgObjs:
# we are not done yet
if dlThread.isAlive():
liveThreadCount += 1
# if one package failed, then we have to fail the entire manifest
threadStatus = dlThread.getStatus()
if (not dlThread.isAlive() and threadStatus.get('progress') != 100):
if isFailedFatal:
raise AgentException(Errors.DC_FAILED_DOWNLOAD,
'failed downloading package (%s) - %s'
% (dlThread.getUriDict().get('uri'), threadStatus.get('errorMsg')))
failed_packages.append(dlThread.getUriDict().get('uri'))
progressTimeout = dlThread.getProgressTimeouts()
if (progressTimeout and timeouts):
timeouts = (timeouts[0] + progressTimeout[0], timeouts[1] + progressTimeout[1])
else:
timeouts = None
pkgProgress = threadStatus.get('progress')
totalProgress += pkgProgress
if (timeouts and timeoutNotSet):
# Setting the timeout once as it is absolute time. Doing this after the timeout for each
# package is available
self.extendTimeout(timeouts[0])
timeoutNotSet = False
LOG.debug('Using overall timeout=%s and progress timeout=%s' % (timeouts[0], timeouts[1]))
# calculate the new progress
# I'm allocating 80 of the progress to be the download
newProgress = calcProgress(1, 79, float(totalProgress) / (100 * len(pkgObjs)))
# now update the pgoress
self._updateProgress(newProgress)
return failed_packages
finally:
# now update the pgoress
self._updateProgress(79)
def createManifest(
testController,
packages=[
'http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/perlserver-1.0.0.unix.cronus'
],
manifest='bar',
service='foo',
createDirs=True):
try:
path = ServiceController.servicePath(service)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'modules'))
os.makedirs(os.path.join(path, '.appdata'))
os.makedirs(os.path.join(path, '.data'))
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
try:
for pkgUri in packages:
pkgDict = PackageUtil.parseUri(pkgUri)
pkgName = pkgDict['package']
localPkgRoot = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'packages')
src = os.path.join(localPkgRoot, pkgName)
shutil.copy(src, pkgDict['packagePath'])
propPkgName = pkgDict['propName']
propSrc = os.path.join(localPkgRoot, propPkgName)
shutil.copy(propSrc, pkgDict['propPath'])
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
body = json.dumps({'package': packages})
response = testController.app.post(
url(controller='manifest',
action='post',
service=service,
manifest=manifest),
headers={'Content-Type': 'application/json'},
params=body)
assert response.status_int == 200, 'Manifest Post assert'
LOG.debug('response body = %s' % response.body)
body = json.loads(response.body)
# wait until the status is done
tm = time.time()
now = tm
while (tm + 120 > now):
response = testController.app.get(body['status'])
body = json.loads(response.body)
LOG.debug("createManifest ********** progress = %s" %
body['progress'])
if (int(body['progress']) == 100):
break
time.sleep(0.1)
now = time.time()
assert tm + 120 >= now, 'Create manifest timed out'
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
LOG.debug('Status response body = %s' % str(body))
assert int(body['progress']) == 100
def __startDownload(self):
""" actual download logic """
try:
LOG.info("Starting package download for package %s" %
self.__uriDict['package'])
# check to see if there's an in progress file,
# since PackageMgr guarantees that duplicate threads will not be spawned
# for same pkg, assume an existing thread was killed.
# attempt to clean up package n move
if (os.path.exists(self.__uriDict['inProgressPackagePath'])):
LOG.debug(
'In progress file (%s) already exists. Will validate and reattempt download if necessary'
% self.__uriDict['inProgressPackagePath'])
if os.path.exists(self.__uriDict['packagePath']):
if (os.path.exists(self.__uriDict['propPath']) and
PackageUtil.validateProp(self.__uriDict['propPath'])
and PackageUtil.validatePackage(
self.__uriDict['packagePath'],
self.__uriDict['propPath'])):
msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict[
'packagePath']
LOG.info(msg)
os.utime(self.__uriDict['packagePath'], None)
os.utime(self.__uriDict['propPath'], None)
self._updateStatus(progress=100)
# NOTE: this is a normal exit not an error!
return
LOG.warning(
'The package already exists. However package prop (%s) failed validation. Downloading package.'
% self.__uriDict['propPath'])
# Delete all traces of package before beginning download
LOG.info('Cleaning up all packages for %s ' %
self.__uriDict['packagePath'])
PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'],
self.__uriDict['packagePath'],
self.__uriDict['propPath'])
AgentThread._updateProgress(self, 0)
if not self.__skipProp:
# First, download .prop file
LOG.info(
'Starting download of prop file %s - %s' %
(self.__uriDict['propUri'], self.__uriDict['propPath']))
self.__download_prop_file()
try:
self.__prop = loadPropFile(self.__uriDict['propPath'])
except FileNotFoundError:
raise AgentException(
Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) unable to read or did not parse' %
(self.__uriDict['propPath']))
AgentThread._updateProgress(self, 2)
self.__setProgressTimeouts()
if self.__uriDict['scheme'] == 'http':
# try download 3 times, with random sleep
for _ in range(3):
try:
sotimeout = float(
pylons.config['download_thread_sotimeout'])
proxies = json.loads(
pylons.config['urlgrabber_proxies'])
urlgrabber.urlgrab(
self.__uriDict['uri'],
self.__uriDict['inProgressPackagePath'],
checkfunc=None if self.__skipProp else
(PackageUtil.validateDownload, (), {}),
progress_obj=DownloadProgress(self),
throttle=float(pylons.config['package_throttle']),
bandwidth=int(pylons.config['package_bandwidth']),
keepalive=0,
timeout=sotimeout,
proxies=proxies)
break
except Exception as exc:
msg = 'Download error %s - %s' % (
str(exc), traceback.format_exc(3))
LOG.warning(msg)
randsleep = randint(30, 60)
time.sleep(randsleep)
else:
# oops! only http and bittorrent supported now
raise AgentException(
Errors.DC_UNSUPPORTED_PROTOCOL,
'Only http protocols is supported at the moment')
self._checkStop()
if not self.__skipProp:
if (not PackageUtil.validatePackage(
self.__uriDict['inProgressPackagePath'],
self.__uriDict['propPath'])):
raise AgentException(
Errors.DC_FAILED_VALIDATE, 'Package ' +
self.__uriDict['packagePath'] + ' failed validation')
os.utime(self.__uriDict['propPath'], None)
utils.rchmod(self.__uriDict['propPath'], "777", 'no')
LOG.info(
'Download complete, will now rename and do validation on this file %s'
% self.__uriDict['packagePath'])
os.rename(self.__uriDict['inProgressPackagePath'],
self.__uriDict['packagePath'])
os.utime(self.__uriDict['packagePath'], None)
utils.rchmod(self.__uriDict['packagePath'], "777", 'no')
LOG.info(
"Download complete, Validation completed, updating progress to 100"
)
self._updateStatus(progress=100)
except AgentException, exc:
self._updateStatus(httpStatus=500,
progress=0,
error=exc.getCode(),
errorMsg=exc.getMsg())
msg = 'Download error %s - %s' % (str(exc),
traceback.format_exc(3))
LOG.error(msg)
raise exc
def test_getNoInstalledPackages(self):
installedPkgPath = 'abcd'
pkgs = PackageUtil.getAllInstalledPackages(installedPkgPath)
self.assertEquals(pkgs.__len__(), 0)
def test_getNoInstalledPackages(self):
installedPkgPath = 'abcd'
pkgs = PackageUtil.getAllInstalledPackages(installedPkgPath)
self.assertEquals(pkgs.__len__(), 0)
def doRun(self):
""" Main body of the thread
Progress Info:
Look at the list of packages this manifest has and figure out the progress of this manifest
Progress 1 = manifest directory created
Progress 2-80 = all packages downloaded
Progress 81-99 = all packages untarred
Progress 100 = all links done
"""
inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(self.__service, self.__manifest))
try:
if self.__service != 'agent':
utils.checkDiskFull()
installedPkgPath = ServiceController.installedPkgPath(self.__service)
# This shouldn'trn happen but make sure there isn'trn already a manifest directory in progress
if (os.path.isdir(inProgressPath)):
# probably another manifest create thread died out half way.
# Cleanup and reattempt manifest creation
LOG.debug('Manifest inprogress found for service/manifest (%s/%s). Will cleanup and retry' % (self.__service, self.__manifest))
shutil.rmtree(inProgressPath)
# make sure that the service path and installed package path exists
if (not os.path.isdir(ServiceController.manifestPath(self.__service)) or
not os.path.isdir(ServiceController.installedPkgPath(self.__service))):
errCode = Errors.SERVICE_NOT_FOUND
msg = 'Service (%s) for manifest (%s) not found' % (self.__service, self.__manifest)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
return
# ok create the manifest path
os.mkdir(inProgressPath)
self._updateProgress(1)
# figure out which of the packages are already there
remainingPackages = {}
for pkgUri in self.__packages:
pkgDict = PackageUtil.parseUri(pkgUri)
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
pkgName = pkgDict['packageName']
if (not os.path.exists(pkgPath)) or ((self.__forcePackages is not None) and pkgName in self.__forcePackages):
remainingPackages[pkgUri] = pkgDict
else:
symlink(pkgPath, os.path.join(inProgressPath, pkgDict['packageName']))
if self.__attemptDownload:
# now make sure all the packages are downloaded
try:
self._downloadPackages(remainingPackages.keys(), skipProp = self.__skipProp)
except AgentException as exc:
# check if it is download error, then modify exception appropriately
if exc.getCode() == Errors.DC_FAILED_DOWNLOAD:
exc = AgentException(Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED,
'Manifest (%s/%s) failed downloading package - %s'
% (self.__service, self.__manifest, exc.getMsg()))
raise exc
else:
if len(remainingPackages) > 0:
raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
'Create Manifest (%s/%s) failed since package is not present and download has been disabled'
% (self.__service, self.__manifest))
LOG.info('Completed download all packages for (%s/%s)' % (self.__service, self.__manifest))
# now untar the packages
import re
pkgSuffix = '.%s' % re.sub(r"\W", "", self.__manifest)
self._untarPackages(remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service), 0, self.__forcePackages, pkgSuffix)
LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__manifest))
# now create the links
for pkgDict in remainingPackages.itervalues():
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
linkPath = os.path.join(inProgressPath, pkgDict['packageName'])
# validate target folder does exist
if not os.path.exists(pkgPath):
raise AgentException(Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath)
symlink(pkgPath, linkPath)
# now move the inProgressPath to the final path
manifestContentPath = ManifestController.manifestContentPath(self.__service, self.__manifest)
os.rename(inProgressPath, ManifestController.manifestPath(self.__service, self.__manifest))
mfContentsFile = file(manifestContentPath, 'w')
for pkgUri in self.__packages:
mfContentsFile.write(('%s%s') % (pkgUri, os.linesep))
mfContentsFile.close()
LOG.info('Completed create manifest directories for (%s/%s)' % (self.__service, self.__manifest))
LOG.info('Completed create manifest for (%s/%s)' % (self.__service, self.__manifest))
self._updateStatus(progress = 100)
except AgentException as exc:
LOG.info(exc.getMsg())
self._updateStatus(httpStatus = 500, error = exc.getCode(), errorMsg = exc.getMsg())
except Exception as exc:
errCode = Errors.UNKNOWN_ERROR
msg = 'Unknown error for (%s/%s) - %s - %s' % (self.__service, self.__manifest,
str(exc), traceback.format_exc(2))
LOG.info(msg)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
finally:
# clean up intermediate progress
try:
shutil.rmtree(inProgressPath)
except OSError:
pass
def doRun(self):
""" Main body of the thread
Progress Info:
Look at the list of packages this manifest has and figure out the progress of this manifest
Progress 1 = manifest directory created
Progress 2-80 = all packages downloaded
Progress 81-99 = all packages untarred
Progress 100 = all links done
"""
try:
if self.__service != 'agent':
utils.checkDiskFull()
installedPkgPath = manifestutil.installedPkgRootPath(self.__service)
# figure out which of the packages are already there
remainingPackages = {}
pkgDict = PackageUtil.parseUri(self.__package)
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
remainingPackages[self.__package] = pkgDict
if (not os.path.exists(pkgPath)):
# now make sure all the packages are downloaded
try:
self._downloadPackages([self.__package])
except AgentException as exc:
# check if it is download error, then modify exception appropriately
if exc.getCode() == Errors.DC_FAILED_DOWNLOAD:
exc = AgentException(Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED,
'Manifest (%s/%s) failed downloading package - %s'
% (self.__service, self.__module, exc.getMsg()))
raise exc
LOG.info('Completed download packages for (%s/%s)' % (self.__service, self.__module))
# now untar the packages
self._untarPackages(remainingPackages, self.__service, ServiceController.installedPkgPath(self.__service))
LOG.info('Completed untar all packages for (%s/%s)' % (self.__service, self.__module))
# now create the links
for pkgDict in remainingPackages.itervalues():
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'], pkgDict['packageVersion'])
modulePath = os.path.join(manifestutil.moduleRootPath(self.__service), self.__module)
# validate target folder does exist
if not os.path.exists(pkgPath):
raise AgentException(Errors.PACKAGE_PATH_ERROR, 'invalid untarred package at %s' % pkgPath)
# remove existing module
isExistingModule = os.path.exists(modulePath)
if isExistingModule:
execThread = self._getBuiltThread('deactivate')
if execThread:
super(ModuleCreate, self)._runExeThread(execThread)
rmlink(modulePath)
execThread = self._getBuiltThread('activate')
if execThread:
super(ModuleCreate, self)._runExeThread(execThread)
symlink(pkgPath, modulePath)
# load controllers from package
manifestutil.processModule(self.__service, self.__module, not isExistingModule)
LOG.info('Completed create module for (%s/%s)' % (self.__service, self.__module))
self._updateStatus(progress = 100)
except AgentException as exc:
LOG.info(exc.getMsg())
self._updateStatus(httpStatus = 500, error = exc.getCode(), errorMsg = exc.getMsg())
except Exception as exc:
errCode = Errors.UNKNOWN_ERROR
msg = 'Unknown error for (%s/%s) - %s - %s' % (self.__service, self.__module,
str(exc), traceback.format_exc(2))
LOG.info(msg)
self._updateStatus(httpStatus = 500, error = errCode, errorMsg = msg)
def run(self):
"""
the garbage collection thread body
go through the list of packages and delete packages over a certain threshold
two variables affect operation
packageMgr_gc_freq = how log GC sleeps for between checks
pacageMgr_package_age = how old a package can get to before it's deleted
go through the list of inprogress threads and remove all threads that are done
"""
#pylint:disable=R0914, R0912, R0915
while (self.__stop == False):
time.sleep(self.__garbageFreq)
try:
# GC in progress hash
self.__inProgressLock.acquire()
try:
for key in self.__inProgressPackages.keys():
if (not self.__inProgressPackages[key].isAlive()):
del self.__inProgressPackages[key]
finally:
self.__inProgressLock.release()
if not self.__stop:
# go through all the packages and see if they are older than the threshold
for filename in glob.glob(
os.path.join(PackageMgr.packagePath(),
'*.cronus.inprogress')):
if (time.time() > os.path.getatime(filename) + float(
pylons.config['packageMgr_package_age'])):
try:
GCLOG.info('Garbage collecting package(%s)' %
filename)
LOG.info('Garbage collecting package(%s)' %
filename)
os.remove(filename)
except OSError, osErr:
GCLOG.error(
'Unable to garbage collect %s - %s' %
(filename, osErr))
LOG.error('Unable to garbage collect %s - %s' %
(filename, osErr))
for filename in glob.glob(
os.path.join(PackageMgr.packagePath(),
'*.cronus')):
LOG.debug(
'garbage collection check in progress for filename %s'
% filename)
try:
if (time.time(
) > os.path.getatime(filename) + float(
pylons.config['packageMgr_package_age'])):
LOG.info('Garbage collecting package(%s)' %
filename)
os.remove(filename)
if os.path.exists(filename + '.prop'):
os.remove(filename + '.prop')
if os.path.exists(filename + '.torrent'):
os.remove(filename + '.torrent')
if os.path.exists(filename + '.inprogress'):
os.remove(filename + '.inprogress')
except OSError, osErr:
GCLOG.error('Unable to garbage collect %s - %s' %
(filename, osErr))
LOG.error('Unable to garbage collect %s - %s' %
(filename, osErr))
#appGlobal = pylons.config['pylons.app_globals']
#import pdb;pdb.set_trace();
if not self.__stop:
PackageUtil.cleanupOrphanedPackages(False)
packageMount = pylons.config['agent_root']
if (not self.__stop
and agenthealth.needAggressiveGC(packageMount)):
PackageUtil.forceCleanUpDownloadedPkgs()
# validate all services and remove rogue services
PackageMgr.cleanRogueService()
def __startDownload(self):
""" actual download logic """
try:
LOG.info("Starting package download for package %s" % self.__uriDict['package'])
# check to see if there's an in progress file,
# since PackageMgr guarantees that duplicate threads will not be spawned
# for same pkg, assume an existing thread was killed.
# attempt to clean up package n move
if (os.path.exists(self.__uriDict['inProgressPackagePath'])):
LOG.debug('In progress file (%s) already exists. Cleanup and reattempt download'
% self.__uriDict['inProgressPackagePath'])
if os.path.exists(self.__uriDict['packagePath']):
if self.__skipProp or ((os.path.exists(self.__uriDict['propPath']) and
PackageUtil.validateProp(self.__uriDict['propPath']) and
PackageUtil.validatePackage(self.__uriDict['packagePath'],
self.__uriDict['propPath']))):
msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict['packagePath']
LOG.info(msg)
os.utime(self.__uriDict['packagePath'], None)
if os.path.exists(self.__uriDict['propPath']):
os.utime(self.__uriDict['propPath'], None)
self._updateStatus(progress = 100)
# NOTE: this is a normal exit not an error!
return
else:
LOG.warning('The package already exists. However package prop (%s) failed validation.'
% self.__uriDict['propPath'])
# Delete all traces of package before beginning download
LOG.debug('Cleaning up all packages for %s ' % self.__uriDict['packagePath'])
PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'],
self.__uriDict['packagePath'],
self.__uriDict['propPath'])
AgentThread._updateProgress(self, 0)
if self.__skipProp:
LOG.info('Skip download of prop file')
else:
# First, download .prop file
LOG.info('Starting download of prop file %s - %s' % (self.__uriDict['propUri'], self.__uriDict['propPath']))
self.__download_prop_file()
try:
self.__prop = loadPropFile(self.__uriDict['propPath'])
except FileNotFoundError:
raise AgentException(Errors.DC_MISSING_PROP_FILE,
'Prop file (%s) unable to read or did not parse' % (self.__uriDict['propPath']))
AgentThread._updateProgress(self, 2)
self.__setProgressTimeouts()
if self.__uriDict['scheme'] == 'http':
# try download 3 times, with random sleep
attempt = configutil.getConfigAsInt('download_thread_attempt')
for _ in range(attempt):
try:
sotimeout = float(pylons.config['download_thread_sotimeout'])
proxies = json.loads(pylons.config['urlgrabber_proxies'])
urlgrabber.urlgrab(self.__uriDict['uri'],
self.__uriDict['inProgressPackagePath'],
# checkfunc = None if self.__skipProp else (PackageUtil.validateDownload, (), {}),
progress_obj = DownloadProgress(self),
throttle = float(pylons.config['package_throttle']),
bandwidth = int(pylons.config['package_bandwidth']),
keepalive = 0,
timeout = sotimeout,
proxies = proxies)
break
except Exception as exc:
msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3))
LOG.warning(msg)
if _ == attempt-1:
raise exc
randsleep = randint(5, 10)
time.sleep(randsleep)
else:
# oops! only http supported now
raise AgentException(Errors.DC_UNSUPPORTED_PROTOCOL, 'Only http protocols is supported at the moment')
self._checkStop()
if self.__skipProp:
LOG.info('Skip validating against prop file')
else:
if (not PackageUtil.validatePackage(self.__uriDict['inProgressPackagePath'],
self.__uriDict['propPath'])):
raise AgentException(Errors.DC_FAILED_VALIDATE, 'Package ' +
self.__uriDict['packagePath'] + ' failed validation')
os.utime(self.__uriDict['propPath'], None)
utils.rchmod(self.__uriDict['propPath'], "777", 'no')
LOG.info('Download complete, rename this file %s' % self.__uriDict['packagePath'])
os.rename(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath'])
os.utime(self.__uriDict['packagePath'], None)
utils.rchmod(self.__uriDict['packagePath'], "777", 'no')
LOG.info("Download complete, updating progress to 100")
self._updateStatus(progress = 100)
except AgentException, exc:
self._updateStatus(httpStatus = 500, progress = 0, error = exc.getCode(), errorMsg = exc.getMsg())
msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3))
LOG.error(msg)
raise exc
def doRun(self):
""" Main body of the thread
Progress Info:
Look at the list of packages this manifest has and figure out the progress of this manifest
Progress 1 = manifest directory created
Progress 2-80 = all packages downloaded
Progress 81-99 = all packages untarred
Progress 100 = all links done
"""
try:
if self.__service != 'agent':
utils.checkDiskFull()
installedPkgPath = manifestutil.installedPkgRootPath(
self.__service)
# figure out which of the packages are already there
remainingPackages = {}
pkgDict = PackageUtil.parseUri(self.__package)
pkgPath = os.path.join(installedPkgPath, pkgDict['packageName'],
pkgDict['packageVersion'])
remainingPackages[self.__package] = pkgDict
if (not os.path.exists(pkgPath)):
# now make sure all the packages are downloaded
try:
self._downloadPackages([self.__package])
except AgentException as exc:
# check if it is download error, then modify exception appropriately
if exc.getCode() == Errors.DC_FAILED_DOWNLOAD:
exc = AgentException(
Errors.MANIFEST_PACKAGE_DOWNLOAD_FAILED,
'Manifest (%s/%s) failed downloading package - %s'
% (self.__service, self.__module, exc.getMsg()))
raise exc
LOG.info('Completed download packages for (%s/%s)' %
(self.__service, self.__module))
# now untar the packages
self._untarPackages(
remainingPackages, self.__service,
ServiceController.installedPkgPath(self.__service))
LOG.info('Completed untar all packages for (%s/%s)' %
(self.__service, self.__module))
# now create the links
for pkgDict in remainingPackages.itervalues():
pkgPath = os.path.join(installedPkgPath,
pkgDict['packageName'],
pkgDict['packageVersion'])
modulePath = os.path.join(
manifestutil.moduleRootPath(self.__service), self.__module)
# validate target folder does exist
if not os.path.exists(pkgPath):
raise AgentException(
Errors.PACKAGE_PATH_ERROR,
'invalid untarred package at %s' % pkgPath)
# remove existing module
isExistingModule = os.path.exists(modulePath)
if isExistingModule:
execThreadTuple = self._getBuiltThread('deactivate')
super(ModuleCreate,
self)._runExeThread(self, execThreadTuple)
rmlink(modulePath)
execThreadTuple = self._getBuiltThread('activate')
super(ModuleCreate, self)._runExeThread(execThreadTuple)
symlink(pkgPath, modulePath)
# load controllers from package
manifestutil.processModule(self.__service, self.__module,
not isExistingModule)
LOG.info('Completed create module for (%s/%s)' %
(self.__service, self.__module))
self._updateStatus(progress=100)
except AgentException as exc:
LOG.info(exc.getMsg())
self._updateStatus(httpStatus=500,
error=exc.getCode(),
errorMsg=exc.getMsg())
except Exception as exc:
errCode = Errors.UNKNOWN_ERROR
msg = 'Unknown error for (%s/%s) - %s - %s' % (
self.__service, self.__manifest, str(exc),
traceback.format_exc(2))
LOG.info(msg)
self._updateStatus(httpStatus=500, error=errCode, errorMsg=msg)
