def post(self, service, manifest):
""" Create a new service object """
from agent.lib.agent_thread.manifest_create import ManifestCreate
try:
LOG.info('Post for service (%s) and manifest (%s) with body: %s',
service, manifest, request.body)
# check to see if the manifest already exists
path = ManifestController.manifestPath(service, manifest)
if (os.path.isdir(path)):
return doneResult(request, response, httpStatus = 201, controller = self)
# parse the body
if (request.body == ""):
return errorResult(request, response, Errors.MANIFEST_PACKAGE_PARSING_ERROR,
'No body found in post command',
controller = self)
body = json.loads(request.body)
packages = body['package']
forcedPackages = body['forcePackageName'] if 'forcePackageName' in body else None
skipProp = asbool(body['skipProp']) if 'skipProp' in body else configutil.getConfigAsBool('download_skip_prop')
LOG.debug('pkgs = %s, %s', packages, forcedPackages)
# parse the package list
for idx, package in enumerate(packages):
# to support reuse of an package from an existing manifest (active if possible)
# without sending the complete package location in request body
if package.startswith('/'):
packageRef = package
tokens = package.split('/')
pkgnamePrefix = tokens[-1].rstrip()
fullPkgLoc = manifestutil.getPackageByName(service, manifest = None, pkgnamePrefix = pkgnamePrefix)
if fullPkgLoc is None:
return errorResult(request, response, Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
'manifest (%s/%s) package (%s) does not exist' %
(service, manifest, packages), controller = self)
else:
LOG.info('expanding package reuse ref %s with full package location %s' % (packageRef, fullPkgLoc))
packages[idx] = fullPkgLoc
appGlobal = config['pylons.app_globals']
# start a thread to create the package
manThread = ManifestCreate(appGlobal.threadMgr, service, manifest, packages, forcePackages = forcedPackages, skipProp = skipProp)
self.injectJobCtx(manThread)
manThread.start()
manThread.threadMgrEvent.wait()
return statusResult(request, response, manThread, controller = self)
except AgentException as excep:
return errorResult(request, response, error = excep.getCode(), errorMsg = excep.getMsg(), controller = self)
except Exception as excep:
return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
errorMsg = 'Unknown error for activateManifest(%s/%s) - %s - %s' %
(service, manifest, str(excep), traceback.format_exc(2)),
controller = self)
def test_post_manifest_inprogress_ok(self):
service = 'foo'
manifest = 'blahblah'
try:
path = ServiceController.servicePath(service)
if os.path.exists(path):
if os.name == 'nt':
cmd = 'rm -r %s' % path
LOG.debug("running command %s" % cmd)
os.system(cmd)
else:
shutil.rmtree(path)
path = ServiceController.manifestPath(service)
os.makedirs(path)
path = ServiceController.installedPkgPath(service)
os.makedirs(path)
inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(service, manifest))
os.makedirs(inProgressPath)
path = ServiceController.downloadedPkgPath(service)
os.makedirs(path)
except Exception as excep:
LOG.warning('got an OS Exception - %s' % str(excep))
createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus"], manifest = 'blahblah', createDirs = False)
self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'blahblah'), 'pkgA')))
def __createManifest(self):
""" create a manifest
"""
service = 'agent'
LOG.info("Create Manifest %s - %s - %s" % (service, self.__manifest, str(self.__packages)))
path = manifestutil.manifestPath(service, self.__manifest)
# check to see if the manifest already exists
if (os.path.isdir(path)):
LOG.info('Manifest %s already exist, skip creating' % self.__manifest)
return
from agent.lib.agent_thread.manifest_create import ManifestCreate
manThread = ManifestCreate(self._threadMgr, service, self.__manifest, self.__packages, skipProp = self.__skipProp)
contextutils.copycontexts(self, manThread, contextutils.CTX_NAMES)
manThread.run()
status = manThread.getStatus()
if (status.has_key('error') and status['error']):
raise AgentException(status['error'], status['errorMsg'])
def get(self, service):
""" Get a new service object """
try:
from agent.lib.agent_thread.manifest_create import ManifestCreate
# make sure the service path exists
path = ServiceController.servicePath(service)
if (not os.path.exists(path)):
return errorResult(request,
response,
error=Errors.SERVICE_NOT_FOUND,
errorMsg='Unable to find service (%s)' %
service,
controller=self)
path = ServiceController.manifestPath(service)
activeManifest = None
manifestList = []
for manifest in os.listdir(path):
if (ManifestCreate.isInProgress(manifest)):
continue
manifestPath = os.path.join(path, manifest)
if (manifest == 'active'):
activeLink = readlink(manifestPath)
if (activeLink == None):
manifestList.append(manifest)
else:
activeManifest = os.path.basename(activeLink)
else:
manifestList.append(manifest)
result = {}
manifestList.sort()
result['manifest'] = manifestList
result['activemanifest'] = activeManifest
return doneResult(request,
response,
result=result,
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error when getting service (%s) %s - %s' %
(service, str(excep), traceback.format_exc(2)),
controller=self)
def get(self, service):
""" Get a new service object """
try:
from agent.lib.agent_thread.manifest_create import ManifestCreate
# make sure the service path exists
path = ServiceController.servicePath(service)
if not os.path.exists(path):
return errorResult(
request,
response,
error=Errors.SERVICE_NOT_FOUND,
errorMsg="Unable to find service (%s)" % service,
controller=self,
)
path = ServiceController.manifestPath(service)
activeManifest = None
manifestList = []
for manifest in os.listdir(path):
if ManifestCreate.isInProgress(manifest):
continue
manifestPath = os.path.join(path, manifest)
if manifest == "active":
activeLink = readlink(manifestPath)
if activeLink == None:
manifestList.append(manifest)
else:
activeManifest = os.path.basename(activeLink)
else:
manifestList.append(manifest)
result = {}
manifestList.sort()
result["manifest"] = manifestList
result["activemanifest"] = activeManifest
return doneResult(request, response, result=result, controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg="Unknown error when getting service (%s) %s - %s"
% (service, str(excep), traceback.format_exc(2)),
controller=self,
)
def __createManifest(self):
""" create a manifest
"""
service = 'agent'
LOG.info("Create Manifest %s - %s - %s" %
(service, self.__manifest, str(self.__packages)))
path = manifestutil.manifestPath(service, self.__manifest)
# check to see if the manifest already exists
if (os.path.isdir(path)):
LOG.info('Manifest %s already exist, skip creating' %
self.__manifest)
return
from agent.lib.agent_thread.manifest_create import ManifestCreate
manThread = ManifestCreate(self._threadMgr,
service,
self.__manifest,
self.__packages,
skipProp=self.__skipProp)
contextutils.copycontexts(self, manThread, contextutils.CTX_NAMES)
manThread.run()
status = manThread.getStatus()
if (status.has_key('error') and status['error']):
raise AgentException(status['error'], status['errorMsg'])
def getManifests(service):
""" return the list of manifests under a specific service """
from agent.lib.agent_thread.manifest_create import ManifestCreate
manifests = []
dirContent = os.listdir(ServiceController.manifestPath(service))
for item in dirContent:
path = os.path.join(ServiceController.manifestPath(service), item)
if (os.path.isdir(path) and not ManifestCreate.isInProgress(path)):
if (not islink(path)):
manifests.append(item)
return sorted(manifests)
def getManifests(service):
""" return the list of manifests under a specific service """
from agent.lib.agent_thread.manifest_create import ManifestCreate
manifests = []
dirContent = os.listdir(ServiceController.manifestPath(service))
for item in dirContent:
path = os.path.join(ServiceController.manifestPath(service), item)
if (os.path.isdir(path) and not ManifestCreate.isInProgress(path)):
if (not islink(path)):
manifests.append(item)
return sorted(manifests)
def cleanupServices():
"""
for all services clean up partially completed manifests
This should only be called during startup.
"""
from agent.lib.agent_thread.manifest_create import ManifestCreate
try:
services = ServiceController.getServices()
for service in services:
manPath = ServiceController.manifestPath(service)
manifestPaths = [os.path.join(manPath, path) for path in os.listdir(manPath)]
[shutil.rmtree(manifestPath) for manifestPath in manifestPaths
if ManifestCreate.isInProgress(os.path.basename(manifestPath))]
except OSError:
pass
def cleanupServices():
"""
for all services clean up partially completed manifests
This should only be called during startup.
"""
from agent.lib.agent_thread.manifest_create import ManifestCreate
try:
services = ServiceController.getServices()
for service in services:
manPath = ServiceController.manifestPath(service)
manifestPaths = [
os.path.join(manPath, path) for path in os.listdir(manPath)
]
[
shutil.rmtree(manifestPath)
for manifestPath in manifestPaths
if ManifestCreate.isInProgress(
os.path.basename(manifestPath))
]
except OSError:
pass
def doRun(self):
""" Main body of the thread """
self._updateProgress(1)
errorMsg = ""
errorCode = None
failed = False
try:
# create manifest if not already exist
mpath = manifestutil.manifestPath(self._service, self._manifest)
if (not os.path.exists(mpath) or not os.path.isdir(mpath)):
self._LOG.debug('pkgs = %s', self.__packages)
# parse the package list
for idx, package in enumerate(self.__packages):
if package.startswith('/'):
packageRef = package
tokens = package.split('/')
pkgnamePrefix = tokens[-1].rstrip()
fullPkgLoc = manifestutil.getPackageFullLocByName(self._service, manifest = None, pkgnamePrefix = pkgnamePrefix)
if fullPkgLoc is None:
raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
'manifest (%s/%s) package (%s) does not exist' %
(self._service, self._manifest, self.__packages))
else:
self._LOG.info('expanding package reuse ref %s with full package location %s' % (packageRef, fullPkgLoc))
self.__packages[idx] = fullPkgLoc
# start a thread to create the package
manThread = ManifestCreate(self._threadMgr, self._service, self._manifest, self.__packages, parentId = self.getUuid())
contextutils.copyJobContexts(self, manThread)
manThread.run()
self._addChildExeThreadId(manThread.getChildExeThreadIds())
status = manThread.getStatus()
if (status['error'] != None):
raise AgentException(status['error'], status['errorMsg'])
self._updateProgress(60)
if (not os.path.exists(mpath)):
raise AgentException(Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Manifest(%s, %s) path missing' % (self._service, self._manifest))
# activate manifest if not already activated
activateThread = ActivateManifest(self._threadMgr, self._service, self._manifest, parentId = self.getUuid())
contextutils.copyJobContexts(self, activateThread)
activateThread.run()
self._addChildExeThreadId(activateThread.getChildExeThreadIds())
status = activateThread.getStatus()
if (status['error'] != None):
raise AgentException(status['error'], status['errorMsg'])
if manifestutil.getActiveManifest(self._service) != self._manifest:
raise AgentException(Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Manifest(%s, %s) path missing' % (self._service, self._manifest))
except SystemExit as exc:
failed = True
if (len(exc.args) == 2):
# ok we got {err code, err msg}
errorCode = exc.args[0]
errorMsg = exc.args[1]
except AgentException as exc:
failed = True
errorMsg = 'Deploy Service - Agent Exception - %s' % exc.getMsg()
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Deploy Service - Unknown error - (%s/%s) - %s - %s' \
% (self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self._LOG.warning(errorMsg)
self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
else:
self._updateProgress(100)
def post(self, service, manifest):
""" Create a new service object """
from agent.lib.agent_thread.manifest_create import ManifestCreate
try:
LOG.info('Post for service (%s) and manifest (%s) with body: %s',
service, manifest, request.body)
# check to see if the manifest already exists
path = ManifestController.manifestPath(service, manifest)
if (os.path.isdir(path)):
return doneResult(request,
response,
httpStatus=201,
controller=self)
# parse the body
if (request.body == ""):
return errorResult(request,
response,
Errors.MANIFEST_PACKAGE_PARSING_ERROR,
'No body found in post command',
controller=self)
body = json.loads(request.body)
packages = body['package']
forcedPackages = body[
'forcePackageName'] if 'forcePackageName' in body else None
skipProp = asbool(
body['skipProp']
) if 'skipProp' in body else configutil.getConfigAsBool(
'download_skip_prop')
LOG.debug('pkgs = %s, %s', packages, forcedPackages)
# parse the package list
for idx, package in enumerate(packages):
# to support reuse of an package from an existing manifest (active if possible)
# without sending the complete package location in request body
if package.startswith('/'):
packageRef = package
tokens = package.split('/')
pkgnamePrefix = tokens[-1].rstrip()
fullPkgLoc = manifestutil.getPackageByName(
service, manifest=None, pkgnamePrefix=pkgnamePrefix)
if fullPkgLoc is None:
return errorResult(
request,
response,
Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
'manifest (%s/%s) package (%s) does not exist' %
(service, manifest, packages),
controller=self)
else:
LOG.info(
'expanding package reuse ref %s with full package location %s'
% (packageRef, fullPkgLoc))
packages[idx] = fullPkgLoc
appGlobal = config['pylons.app_globals']
# start a thread to create the package
manThread = ManifestCreate(appGlobal.threadMgr,
service,
manifest,
packages,
forcePackages=forcedPackages,
skipProp=skipProp)
self.injectJobCtx(manThread)
manThread.start()
manThread.threadMgrEvent.wait()
return statusResult(request, response, manThread, controller=self)
except AgentException as excep:
return errorResult(request,
response,
error=excep.getCode(),
errorMsg=excep.getMsg(),
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error for activateManifest(%s/%s) - %s - %s'
% (service, manifest, str(excep), traceback.format_exc(2)),
controller=self)
def doRun(self):
""" Main body of the thread """
spath = manifestutil.servicePath(self._service)
self._updateProgress(1)
errorMsg = ""
errorCode = None
failed = False
ctxNames = ['guid', 'service']
try:
# create service if not already exist
if not os.path.exists(spath):
os.makedirs(spath)
os.makedirs(os.path.join(spath, 'manifests'))
os.makedirs(os.path.join(spath, 'installed-packages'))
os.makedirs(os.path.join(spath, 'modules'))
os.makedirs(os.path.join(spath, 'downloaded-packages'))
os.makedirs(os.path.join(spath, '.appdata'))
os.makedirs(os.path.join(spath, '.data'))
import pwd
uname = pylons.config['app_user_account']
uid = pwd.getpwnam(uname).pw_uid
gid = pwd.getpwnam(uname).pw_gid
utils.rchown(os.path.join(spath, '.appdata'), uid, gid)
# verify that the path exists
if (not os.path.isdir(spath)):
raise AgentException(Errors.UNKNOWN_ERROR, "Service(%s) was not created" % self._service)
self._updateProgress(20)
# create manifest if not already exist
mpath = manifestutil.manifestPath(self._service, self._manifest)
if (not os.path.exists(mpath) or not os.path.isdir(mpath)):
self.__LOG.debug('pkgs = %s', self.__packages)
# parse the package list
for idx, package in enumerate(self.__packages):
if package.startswith('/'):
packageRef = package
tokens = package.split('/')
pkgnamePrefix = tokens[-1].rstrip()
fullPkgLoc = manifestutil.getPackageByName(self._service, manifest = None, pkgnamePrefix = pkgnamePrefix)
if fullPkgLoc is None:
raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
'manifest (%s/%s) package (%s) does not exist' %
(self._service, self._manifest, self.__packages))
else:
self.__LOG.info('expanding package reuse ref %s with full package location %s' % (packageRef, fullPkgLoc))
self.__packages[idx] = fullPkgLoc
# start a thread to create the package
manThread = ManifestCreate(threadmgr.NULL_THREADMGR, self._service, self._manifest, self.__packages, skipProp = self.__skipProp)
contextutils.copycontexts(self, manThread, ctxNames)
manThread.run()
status = manThread.getStatus()
if (status['error'] != None):
raise AgentException(status['error'], status['errorMsg'])
self._updateProgress(60)
if (not os.path.exists(mpath)):
raise AgentException(Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Manifest(%s, %s) path missing' % (self._service, self._manifest))
if not self.__skipActivation:
activateThread = ActivateManifest(threadmgr.NULL_THREADMGR, self._service, self._manifest)
contextutils.copycontexts(self, activateThread, ctxNames)
activateThread.run()
status = activateThread.getStatus()
if (status['error'] != None):
raise AgentException(status['error'], status['errorMsg'])
# activte manifest if not already activated
if manifestutil.getActiveManifest(self._service) != self._manifest:
raise AgentException(Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
'Manifest(%s, %s) path missing' % (self._service, self._manifest))
except SystemExit as exc:
failed = True
if (len(exc.args) == 2):
# ok we got {err code, err msg}
errorCode = exc.args[0]
errorMsg = exc.args[1]
except AgentException as exc:
failed = True
errorMsg = 'Deploy Service - Agent Exception - %s' % exc.getMsg()
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Deploy Service - Unknown error - (%s/%s) - %s - %s' \
% (self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.warning(errorMsg)
self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
else:
self._updateProgress(100)