def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
activeManifest = None
try:
activePath = manifestutil.manifestPath(self._service, 'active')
# make sure that the active path exists and it is a link
# Should we check this again since we already have a check in action controller
if not os.path.exists(activePath) or not islink(activePath):
raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, errorMsg = 'No active manifest - cannot restart service')
activeManifest = os.path.basename(readlink(activePath))
self.__lcmActionManifest(self._service, activeManifest, self.__action)
self._LOG.info('Done: %s service for (%s/%s)' % (self.__action, self._service, activeManifest))
self._updateStatus(progress = 100)
except AgentException as exc:
failed = True
errorMsg = '%s Service - Agent Exception - %s' % (self.__action, exc.getMsg())
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = '%s Service - Unknown error - (%s/%s) - %s - %s' \
% (self.__action, self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self._LOG.error(errorMsg)
self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
def get(self, module):
""" Get a new service object """
try:
# make sure the service path exists
path = manifestutil.modulePath("agent", module)
if not os.path.exists(path):
return errorResult(
request,
response,
error=Errors.SERVICE_NOT_FOUND,
errorMsg="Unable to find module (%s)" % module,
controller=self,
)
result = {}
modulePackage = readlink(path)
result["package"] = modulePackage
return doneResult(request, response, result=result, controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg="Unknown error when getting module (%s) %s - %s"
% (module, str(excep), traceback.format_exc(2)),
controller=self,
)
def getAllSymLinks(service):
""" return all the symlinks from manifests to packages for a given service"""
LOG.debug('calling getAllSymLinks %s' % service)
linkedPaths = []
manPath = ServiceController.manifestPath(service)
LOG.debug('manifestPath is %s' % manPath)
for path in os.listdir(manPath):
LOG.debug('path is %s' % path)
if not islink(os.path.join(manPath, path)):
LOG.debug('path is dir not a link')
pkgPaths = [
packageDir
for packageDir in os.listdir(os.path.join(manPath, path))
]
LOG.debug('pkgPaths is %s' % pkgPaths)
for pkgPath in pkgPaths:
try:
LOG.debug('pkgPath is %s' % pkgPath)
if not os.path.isfile(
os.path.join(
manPath, path, pkgPath)) and islink(
os.path.join(manPath, path, pkgPath)):
LOG.debug('pkgPaths is %s' % pkgPath)
targetPath = os.path.abspath(
readlink(os.path.join(manPath, path, pkgPath)))
linkedPaths.append(targetPath)
LOG.debug('targetPath is %s ' % targetPath)
except BaseException as exc:
LOG.error('failed to read link for the pkg path %s' %
str(exc))
return linkedPaths
def delete(self, service, manifest):
""" Delete a new service object """
try:
path = manifestutil.manifestPath(service, manifest)
if (not os.path.isdir(path)):
return errorResult(request, response, Errors.MANIFEST_NOT_FOUND,
'manifest (%s/%s) missing service' % (service, manifest),
controller=self)
# first check that this isn't the active manifest
path = manifestutil.manifestPath(service)
if (os.path.exists(path)):
activePath = os.path.basename(readlink(path))
deletePath = os.path.basename(manifestutil.manifestPath(service, manifest))
if (activePath == deletePath):
return errorResult(request, response, Errors.MANIFEST_DELETING_ACTIVE_MANIFEST,
'Manifest(%s, %s) attempting to delete active manifest'
% (service, manifest),
controller=self)
# now try to delete the manifest directory
appGlobal = config['pylons.app_globals']
manThread = ManifestDelete(appGlobal.threadMgr, service, manifest)
self.injectJobCtx(manThread)
manThread.start()
manThread.threadMgrEvent.wait()
return statusResult(request, response, manThread, controller=self)
except Exception as excep:
return errorResult(request, response, error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error for delete manifest(%s/%s) - %s - %s' %
(service, manifest, str(excep), traceback.format_exc(2)),
controller=self)
def get(self, module):
""" Get a new service object """
try:
# make sure the service path exists
path = manifestutil.modulePath('agent', module)
if (not os.path.exists(path)):
return errorResult(request,
response,
error=Errors.SERVICE_NOT_FOUND,
errorMsg='Unable to find module (%s)' %
module,
controller=self)
result = {}
modulePackage = readlink(path)
result['package'] = modulePackage
return doneResult(request,
response,
result=result,
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error when getting module (%s) %s - %s' %
(module, str(excep), traceback.format_exc(2)),
controller=self)
def activateManifest(testController, manifest = 'bar', service = 'foo'):
body = json.dumps({'manifest':manifest})
response = testController.app.post(url(controller = 'manifest', action = 'activate', service = service, manifest = manifest),
headers = {'Content-Type' : 'application/json'},
params = body)
assert response.status_int == 200, 'Action get assert'
body = json.loads(response.body)
tm = time.time()
while (tm + 120 > time.time()):
response = testController.app.get(body['status'])
body = json.loads(response.body)
LOG.debug("activateManifest ********** progress = %s" % body['progress'])
if (int(body['progress']) == 100):
break
time.sleep(0.1)
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
LOG.debug ('Status response body = %s' % str(body))
assert body['progress'] == 100
# let's make sure the link is there correctly
activePath = os.path.join(ServiceController.manifestPath(service), 'active')
LOG.debug ('active path = ' + activePath)
assert islink(activePath)
link = readlink(activePath)
LOG.debug ('link = ' + link)
assert link == manifest
def activeManifestPath(service):
""" compute the path to this __service active manifest """
activePath = manifestPath(service, ACTIVE_MANIFEST)
if (not os.path.exists(activePath)):
return ''
realManifest = readlink(activePath)
return realManifest
def activeManifestPath(service):
""" compute the path to this __service active manifest """
activePath = manifestPath(service, ACTIVE_MANIFEST)
if (not os.path.exists(activePath)):
return ''
realManifest = readlink(activePath)
return realManifest
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
try:
activePath = os.path.join(
ServiceController.manifestPath(self._service), 'active')
oldManifest = None
# make sure that if the active path exists, it's a link
# if not log that and delete the link
if (os.path.exists(activePath) and not os.name == 'nt'
and not islink(activePath)):
self.__LOG.error('%s is not a link. Attempted to delete' %
activePath)
shutil.rmtree(activePath)
if (os.path.exists(activePath)):
oldManifest = os.path.basename(readlink(activePath))
else:
raise AgentException(
error=Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
errorMsg='No active manifest - cannot deactivate service')
self.__deactivateManifest(self._service, oldManifest)
self.__removeSymlink(self._service)
except SystemExit as exc:
failed = True
if (len(exc.args) == 2):
# ok we got {err code, err msg}
errorCode = exc.args[0]
errorMsg = exc.args[1]
raise exc
except AgentException as exc:
failed = True
errorMsg = 'Deactivate Manifest - Agent Exception - %s' % exc.getMsg(
)
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Deactivate Manifest - Unknown error - (%s) - %s - %s' \
% (self._service, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.warning(errorMsg)
self._updateStatus(httpStatus=500,
error=errorCode,
errorMsg=errorMsg)
self.__LOG.debug('Done: activate manifest for (%s)' %
(self._service))
self._updateProgress(100)
def getActiveManifestPath(service):
"""
return the name of the active manifest under a specific service
@param service: name of service
@return: path of active manifest, or empty string if no active manifest
"""
activePath = os.path.join(ServiceController.manifestPath(service), 'active')
if (not os.path.exists(activePath)):
return ''
return readlink(activePath)
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
activeManifest = None
try:
activePath = manifestutil.manifestPath(self._service, 'active')
# make sure that the active path exists and it is a link
# Should we check this again since we already have a check in action controller
if not os.path.exists(activePath) or not islink(activePath):
raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
errorMsg = 'Service %s has no active manifest' % self._service)
activeManifest = os.path.basename(readlink(activePath))
if self.__action == StartStopService.ACTION_SHUTDOWN:
self._shutdownManifest(self._service, activeManifest, 50, 90)
elif self.__action == StartStopService.ACTION_STARTUP:
self._startupManifest(self._service, activeManifest, 50, 90)
elif self.__action == StartStopService.ACTION_RESTART:
self._restartManifest(self._service, activeManifest, 10, 90)
else:
raise AgentException(error = Errors.INVALID_LIFECYCLE_ACTION,
errorMsg = 'Invalid life cycle action - %s' % self.__action)
self._LOG.info('Done: %s service for (%s/%s)' % (self.__action, self._service, activeManifest))
self._updateStatus(progress = 100)
except AgentException as exc:
failed = True
errorMsg = '%s Service - Agent Exception - %s' % (self.__action, exc.getMsg())
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = '%s Service - Unknown error - (%s/%s) - %s - %s' \
% (self.__action, self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self._LOG.error(errorMsg)
if not self._skipCleanupOnFailure() and self.__action != StartStopService.ACTION_SHUTDOWN and self._service and activeManifest:
try:
self._LOG.info('%s Service %s failed, shutdown to cleanup' % (self.__action, self._service))
self._shutdownManifest(self._service, activeManifest, 91, 99)
except BaseException as excep:
self._LOG.error('Cleanup failed - %s' % str(excep))
self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
def getActiveManifestPath(service):
"""
return the name of the active manifest under a specific service
@param service: name of service
@return: path of active manifest, or empty string if no active manifest
"""
activePath = os.path.join(ServiceController.manifestPath(service),
'active')
if (not os.path.exists(activePath)):
return ''
return readlink(activePath)
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
activeManifest = None
try:
activePath = os.path.join(
ServiceController.manifestPath(self._service), 'active')
# make sure that the active path exists and it is a link
# Should we check this again since we already have a check in action controller
if not os.path.exists(activePath) or not islink(activePath):
raise AgentException(
error=Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
errorMsg='No active manifest - cannot reset service')
activeManifest = os.path.basename(readlink(activePath))
self.__shutdownManifest(self._service, activeManifest)
self.__deactivateManifest(self._service, activeManifest)
self.__activateManifest(self._service, activeManifest)
self.__startupManifest(self._service, activeManifest)
self.__LOG.info('Done: reset service for (%s/%s)' %
(self._service, activeManifest))
self._updateStatus(progress=100)
except AgentException as exc:
failed = True
errorMsg = 'Activate Manifest - Agent Exception - %s' % exc.getMsg(
)
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Activate Manifest - Unknown error - (%s/%s) - %s - %s' \
% (self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.error(errorMsg)
if self._service and activeManifest:
try:
self.__LOG.info(
'Reset service %s failed, shutdown to cleanup' %
self._service)
self.__shutdownManifest(self._service, activeManifest)
except BaseException as excep:
self.__LOG.error('Cleanup failed - %s' % str(excep))
self._updateStatus(httpStatus=500,
error=errorCode,
errorMsg=errorMsg)
def get(self, service):
""" Get a new service object """
try:
from agent.lib.agent_thread.manifest_create import ManifestCreate
# make sure the service path exists
path = ServiceController.servicePath(service)
if (not os.path.exists(path)):
return errorResult(request,
response,
error=Errors.SERVICE_NOT_FOUND,
errorMsg='Unable to find service (%s)' %
service,
controller=self)
path = ServiceController.manifestPath(service)
activeManifest = None
manifestList = []
for manifest in os.listdir(path):
if (ManifestCreate.isInProgress(manifest)):
continue
manifestPath = os.path.join(path, manifest)
if (manifest == 'active'):
activeLink = readlink(manifestPath)
if (activeLink == None):
manifestList.append(manifest)
else:
activeManifest = os.path.basename(activeLink)
else:
manifestList.append(manifest)
result = {}
manifestList.sort()
result['manifest'] = manifestList
result['activemanifest'] = activeManifest
return doneResult(request,
response,
result=result,
controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error when getting service (%s) %s - %s' %
(service, str(excep), traceback.format_exc(2)),
controller=self)
def get(self, service):
""" Get a new service object """
try:
from agent.lib.agent_thread.manifest_create import ManifestCreate
# make sure the service path exists
path = ServiceController.servicePath(service)
if not os.path.exists(path):
return errorResult(
request,
response,
error=Errors.SERVICE_NOT_FOUND,
errorMsg="Unable to find service (%s)" % service,
controller=self,
)
path = ServiceController.manifestPath(service)
activeManifest = None
manifestList = []
for manifest in os.listdir(path):
if ManifestCreate.isInProgress(manifest):
continue
manifestPath = os.path.join(path, manifest)
if manifest == "active":
activeLink = readlink(manifestPath)
if activeLink == None:
manifestList.append(manifest)
else:
activeManifest = os.path.basename(activeLink)
else:
manifestList.append(manifest)
result = {}
manifestList.sort()
result["manifest"] = manifestList
result["activemanifest"] = activeManifest
return doneResult(request, response, result=result, controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg="Unknown error when getting service (%s) %s - %s"
% (service, str(excep), traceback.format_exc(2)),
controller=self,
)
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
activeManifest = None
try:
activePath = manifestutil.manifestPath(self._service, 'active')
# make sure that the active path exists and it is a link
# Should we check this again since we already have a check in action controller
if not os.path.exists(activePath) or not islink(activePath):
raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, errorMsg = 'No active manifest - cannot restart service')
activeManifest = os.path.basename(readlink(activePath))
if self.__action == StartStopService.ACTION_SHUTDOWN:
self.__shutdownManifest(self._service, activeManifest)
elif self.__action == StartStopService.ACTION_STARTUP:
self.__startupManifest(self._service, activeManifest)
elif self.__action == StartStopService.ACTION_RESTART:
self.__restartManifest(self._service, activeManifest)
elif self.__action == StartStopService.ACTION_REBOOT:
self.__rebootManifest(self._service, activeManifest)
else:
raise AgentException(error = Errors.INVALID_LIFECYCLE_ACTION, errorMsg = 'Invalid life cycle action - %s' % self.__action)
self.__LOG.info('Done: %s service for (%s/%s)' % (self.__action, self._service, activeManifest))
self._updateStatus(progress = 100)
except AgentException as exc:
failed = True
errorMsg = '%s Service - Agent Exception - %s' % (self.__action, exc.getMsg())
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = '%s Service - Unknown error - (%s/%s) - %s - %s' \
% (self.__action, self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.error(errorMsg)
if not self._skipCleanupOnFailure() and self.__action != StartStopService.ACTION_SHUTDOWN and self._service and activeManifest:
try:
self.__LOG.info('%s Service %s failed, shutdown to cleanup' % (self.__action, self._service))
self.__shutdownManifest(self._service, activeManifest)
except BaseException as excep:
self.__LOG.error('Cleanup failed - %s' % str(excep))
self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
def getLocalPyPkg():
"""
reuse of local python package instead of download it again from source of truth,
this should be the common use case for selfupdate without needing to update the python package
"""
activeManifest = manifestutil.getActiveManifest('agent')
activePyLink = os.path.join(manifestutil.manifestPath('agent', activeManifest), 'python_package')
if (os.path.exists(activePyLink)):
activePyPath = readlink(activePyLink)
pyVersion = os.path.basename(activePyPath)
pyPkgName = ('python_package-%s' % pyVersion)
if AgentUpdate.nameRe.match(pyPkgName):
return ('http://localhost:12020/%s.cronus' % pyPkgName)
else:
raise AgentException(Errors.PACKAGE_SCHEME_ERROR, 'package name %s is not valid' % pyPkgName)
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
try:
activePath = os.path.join(ServiceController.manifestPath(self._service), 'active')
oldManifest = None
# make sure that if the active path exists, it's a link
# if not log that and delete the link
if (os.path.exists(activePath) and not os.name == 'nt' and not islink(activePath)):
self.__LOG.error('%s is not a link. Attempted to delete' % activePath)
shutil.rmtree(activePath)
if (os.path.exists(activePath)):
oldManifest = os.path.basename(readlink(activePath))
else:
raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, errorMsg = 'No active manifest - cannot deactivate service')
self.__deactivateManifest(self._service, oldManifest)
self.__removeSymlink(self._service)
except SystemExit as exc:
failed = True
if (len(exc.args) == 2):
# ok we got {err code, err msg}
errorCode = exc.args[0]
errorMsg = exc.args[1]
raise exc
except AgentException as exc:
failed = True
errorMsg = 'Deactivate Manifest - Agent Exception - %s' % exc.getMsg()
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Deactivate Manifest - Unknown error - (%s) - %s - %s' \
% (self._service, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.warning(errorMsg)
self._updateStatus(httpStatus = 500, error = errorCode,
errorMsg = errorMsg)
self.__LOG.debug('Done: activate manifest for (%s)' % (self._service))
self._updateProgress(100)
def getModuleSymLinks(service = 'agent'):
""" return all the symlinks from manifests to packages for a given service"""
LOG.debug('calling getAllSymLinks %s' % service)
linkedPaths = []
mRootPath = moduleRootPath(service)
LOG.debug('moduleRootPath is %s' % mRootPath)
for pkgPath in os.listdir(mRootPath):
try:
LOG.debug('pkgPath is %s' % pkgPath)
if not os.path.isfile(os.path.join(mRootPath, pkgPath)) and islink(os.path.join(mRootPath, pkgPath)):
targetPath = os.path.abspath(readlink(os.path.join(mRootPath, pkgPath)))
linkedPaths.append(targetPath)
LOG.debug('targetPath is %s ' % targetPath)
except BaseException as exc:
LOG.error('failed to read link for the pkg path %s' % str(exc))
return linkedPaths
def getModuleSymLinks(service = 'agent'):
""" return all the symlinks from manifests to packages for a given service"""
LOG.debug('calling getAllSymLinks %s' % service)
linkedPaths = []
mRootPath = moduleRootPath(service)
LOG.debug('moduleRootPath is %s' % mRootPath)
for pkgPath in os.listdir(mRootPath):
try:
LOG.debug('pkgPath is %s' % pkgPath)
if not os.path.isfile(os.path.join(mRootPath, pkgPath)) and islink(os.path.join(mRootPath, pkgPath)):
targetPath = os.path.abspath(readlink(os.path.join(mRootPath, pkgPath)))
linkedPaths.append(targetPath)
LOG.debug('targetPath is %s ' % targetPath)
except BaseException as exc:
LOG.error('failed to read link for the pkg path %s' % str(exc))
return linkedPaths
def delete(self, service, manifest):
""" Delete a new service object """
try:
path = ManifestController.manifestPath(service, manifest)
if (not os.path.isdir(path)):
return errorResult(request,
response,
Errors.MANIFEST_NOT_FOUND,
'manifest (%s/%s) missing service' %
(service, manifest),
controller=self)
# first check that this isn't the active manifest
path = os.path.join(ServiceController.manifestPath(service),
'active')
if (os.path.exists(path)):
activePath = os.path.basename(readlink(path))
deletePath = os.path.basename(
ManifestController.manifestPath(service, manifest))
if (activePath == deletePath):
return errorResult(
request,
response,
Errors.MANIFEST_DELETING_ACTIVE_MANIFEST,
'Manifest(%s, %s) attempting to delete active manifest'
% (service, manifest),
controller=self)
# now try to delete the manifest directory
appGlobal = config['pylons.app_globals']
manThread = ManifestDelete(appGlobal.threadMgr, service, manifest)
self.injectJobCtx(manThread)
manThread.start()
manThread.threadMgrEvent.wait()
return statusResult(request, response, manThread, controller=self)
except Exception as excep:
return errorResult(
request,
response,
error=Errors.UNKNOWN_ERROR,
errorMsg='Unknown error for delete manifest(%s/%s) - %s - %s' %
(service, manifest, str(excep), traceback.format_exc(2)),
controller=self)
def get(self, service, manifest):
""" Get a new service object """
LOG.info('Get for service (%s) and manifest (%s)', service, manifest)
try:
# first check that the manifest directory exists
path = ManifestController.manifestPath(service, manifest)
if (not os.path.isdir(path)):
return errorResult(request,
response,
Errors.MANIFEST_NOT_FOUND,
'manifest (%s/%s) missing service' %
(service, manifest),
controller=self)
# now go through the list of packages in the manifest
packages = []
packageLinkNames = glob.glob(
os.path.join(self.manifestPath(service, manifest), '*'))
for packageLink in packageLinkNames:
package = readlink(packageLink)
LOG.debug('Get package (%s) in manifest (%s)', package,
manifest)
# deal with the case where the package has a / or \ (for windoz) at the end
package = package.rstrip('/')
package = package.rstrip('\\')
# the name of the package can be constructed from the last two path components
(head, version) = os.path.split(package)
(head, name) = os.path.split(head)
LOG.debug('Add package %s-%s.cronus' % (name, version))
packages.append('%s-%s.cronus' % (name, version))
except OSError as excp:
return errorResult(request,
response,
Errors.MANIFEST_PATH_ERROR,
'Manifest(%s, %s) path error: %s' %
(service, manifest, str(excp)),
controller=self)
return doneResult(request, response, result=packages, controller=self)
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
failed = False
activeManifest = None
try:
activePath = os.path.join(ServiceController.manifestPath(self._service), 'active')
# make sure that the active path exists and it is a link
# Should we check this again since we already have a check in action controller
if not os.path.exists(activePath) or not islink(activePath):
raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, errorMsg = 'No active manifest - cannot reset service')
activeManifest = os.path.basename(readlink(activePath))
self.__shutdownManifest(self._service, activeManifest)
self.__deactivateManifest(self._service, activeManifest)
self.__activateManifest(self._service, activeManifest)
self.__startupManifest(self._service, activeManifest)
self.__LOG.info('Done: reset service for (%s/%s)' % (self._service, activeManifest))
self._updateStatus(progress = 100)
except AgentException as exc:
failed = True
errorMsg = 'Activate Manifest - Agent Exception - %s' % exc.getMsg()
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Activate Manifest - Unknown error - (%s/%s) - %s - %s' \
% (self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
self.__LOG.error(errorMsg)
if self._service and activeManifest:
try:
self.__LOG.info('Reset service %s failed, shutdown to cleanup' % self._service)
self.__shutdownManifest(self._service, activeManifest)
except BaseException as excep:
self.__LOG.error('Cleanup failed - %s' % str(excep))
self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
def getLocalPyPkg():
"""
reuse of local python package instead of download it again from source of truth,
this should be the common use case for selfupdate without needing to update the python package
"""
activeManifest = manifestutil.getActiveManifest('agent')
activePyLink = os.path.join(
manifestutil.manifestPath('agent', activeManifest),
'python_package')
if (os.path.exists(activePyLink)):
activePyPath = readlink(activePyLink)
pyVersion = os.path.basename(activePyPath)
pyPkgName = ('python_package-%s' % pyVersion)
if AgentUpdate.nameRe.match(pyPkgName):
return ('http://localhost:12020/%s.cronus' % pyPkgName)
else:
raise AgentException(
Errors.PACKAGE_SCHEME_ERROR,
'package name %s is not valid' % pyPkgName)
def activateManifest(testController, manifest='bar', service='foo'):
body = json.dumps({'manifest': manifest})
response = testController.app.post(
url(controller='action', action='activatemanifest', service=service),
headers={'Content-Type': 'application/json'},
params=body)
assert response.status_int == 200, 'Action get assert'
body = json.loads(response.body)
tm = time.time()
while (tm + 120 > time.time()):
response = testController.app.get(body['status'])
body = json.loads(response.body)
LOG.debug("activateManifest ********** progress = %s" %
body['progress'])
if (int(body['progress']) == 100):
break
time.sleep(0.1)
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
LOG.debug('Status response body = %s' % str(body))
assert body['progress'] == 100
# let's make sure the link is there correctly
activePath = os.path.join(ServiceController.manifestPath(service),
'active')
LOG.debug('active path = ' + activePath)
assert islink(activePath)
link = readlink(activePath)
LOG.debug('link = ' + link)
if os.name == 'nt':
manifestPath = os.path.join(ServiceController.manifestPath(service),
manifest)
assert (link == manifestPath or link == manifest)
else:
assert link == manifest
def get(self, service, manifest):
""" Get a new service object """
LOG.info('Get for service (%s) and manifest (%s)', service, manifest)
try:
# first check that the manifest directory exists
path = manifestutil.manifestPath(service, manifest)
if (not os.path.isdir(path)):
return errorResult(request, response, Errors.MANIFEST_NOT_FOUND,
'manifest (%s/%s) missing service' % (service, manifest),
controller=self)
# now go through the list of packages in the manifest
packages = []
packageLinkNames = glob.glob(os.path.join(manifestutil.manifestPath(service, manifest), '*'))
for packageLink in packageLinkNames:
package = readlink(packageLink)
LOG.debug('Get package (%s) in manifest (%s)', package, manifest)
# deal with the case where the package has a / or \ (for windoz) at the end
package = package.rstrip('/')
package = package.rstrip('\\')
# the name of the package can be constructed from the last two path components
(head, version) = os.path.split(package)
(head, name) = os.path.split(head)
LOG.debug('Add package %s-%s.cronus' % (name, version))
packages.append('%s-%s.cronus' % (name, version))
except OSError as excp:
return errorResult(request, response, Errors.MANIFEST_PATH_ERROR,
'Manifest(%s, %s) path error: %s' % (service, manifest, str(excp)),
controller=self)
return doneResult(request, response, result=packages, controller=self)
def getAllSymLinks(service):
""" return all the symlinks from manifests to packages for a given service"""
LOG.debug('calling getAllSymLinks %s' % service)
linkedPaths = []
manPath = manifestRootPath(service)
LOG.debug('manifestPath is %s' % manPath)
for path in os.listdir(manPath):
LOG.debug('path is %s' % path)
if not islink(os.path.join(manPath, path)):
LOG.debug('path is dir not a link')
pkgPaths = [ packageDir for packageDir in os.listdir(os.path.join(manPath, path)) ]
LOG.debug('pkgPaths is %s' % pkgPaths)
for pkgPath in pkgPaths:
try:
LOG.debug('pkgPath is %s' % pkgPath)
if not os.path.isfile(os.path.join(manPath, path, pkgPath)) and islink(os.path.join(manPath, path, pkgPath)):
LOG.debug('pkgPaths is %s' % pkgPath)
targetPath = os.path.abspath(readlink(os.path.join(manPath, path, pkgPath)))
linkedPaths.append(targetPath)
LOG.debug('targetPath is %s ' % targetPath)
except BaseException as exc:
LOG.error('failed to read link for the pkg path %s' % str(exc))
return linkedPaths
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
symlinkSwitched = False
failed = False
try:
activePath = os.path.join(ServiceController.manifestPath(self._service), 'active')
oldManifest = None
appGlobal = pylons.config['pylons.app_globals']
# make sure that if the active path exists, it's a link
# if not log that and delete the link
if (os.path.exists(activePath) and not os.name == 'nt' and not islink(activePath)):
self.__LOG.error('%s is not a link. Attempted to delete' % activePath)
shutil.rmtree(activePath)
if (os.path.exists(activePath)):
oldManifest = os.path.basename(readlink(activePath))
self.__installManifest(self._service, self._manifest)
self.__deactivateManifest(self._service, oldManifest)
symlinkSwitched = self.__switchSymlink(self._service, self._manifest)
appGlobal = pylons.config['pylons.app_globals']
if self._service == 'agent':
# START Jeff 09 19 2012: save the timestamp and old manifestPath to .recover: only for agent service.
recoveryInfo = {}
beforeKillTimeHuman = strftime("%Y-%m-%d %H:%M:%S", localtime())
beforeKillTimeStamp = int(time.time())
recoveryInfo['beforeKillTimeHuman'] = str(beforeKillTimeHuman)
recoveryInfo['beforeKillTimeStamp'] = str(beforeKillTimeStamp)
# make sure there is always quote: even if null. easy for json parsing in startup
if self._manifest:
recoveryInfo['newManifest'] = self._manifest
else:
recoveryInfo['newManifest'] = 'null'
if oldManifest:
recoveryInfo['oldManifest'] = oldManifest
else:
recoveryInfo['oldManifest'] = 'null'
manifestutil.writeJson('agent', '.recovery', recoveryInfo)
# END Jeff 09 19 2012: save the timestamp and .recover files with
# persist threads result before shutting down
if (hasattr(appGlobal, 'threadMgr') and appGlobal.threadMgr != None):
killStatus = {}
killStatus['httpStatus'] = 500
killStatus['error'] = Errors.THREAD_KILLED_AGENT_RESTART
killStatus['errorMsg'] = 'thread killed, agent restart'
appGlobal.threadMgr.snapshot(killStatus, True)
self.__activateManifest(self._service, self._manifest)
except SystemExit as exc:
failed = True
if (len(exc.args) == 2):
# ok we got {err code, err msg}
errorCode = exc.args[0]
errorMsg = exc.args[1]
raise exc
except AgentException as exc:
failed = True
errorMsg = 'Activate Manifest - Agent Exception - %s' % exc.getMsg()
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Activate Manifest - Unknown error - (%s/%s) - %s - %s' \
% (self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
if not self._skipCleanupOnFailure():
self.__cleanup(symlinkSwitched, errorMsg, errorCode)
self.__LOG.warning(errorMsg)
self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
else:
self._updateProgress(100)
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
symlinkSwitched = False
failed = False
try:
activePath = os.path.join(
ServiceController.manifestPath(self._service), 'active')
oldManifest = None
appGlobal = pylons.config['pylons.app_globals']
# make sure that if the active path exists, it's a link
# if not log that and delete the link
if (os.path.exists(activePath) and not os.name == 'nt'
and not islink(activePath)):
self.__LOG.error('%s is not a link. Attempted to delete' %
activePath)
shutil.rmtree(activePath)
if (os.path.exists(activePath)):
oldManifest = os.path.basename(readlink(activePath))
self.__installManifest(self._service, self._manifest)
self.__deactivateManifest(self._service, oldManifest)
symlinkSwitched = self.__switchSymlink(self._service,
self._manifest)
appGlobal = pylons.config['pylons.app_globals']
if self._service == 'agent':
# START Jeff 09 19 2012: save the timestamp and old manifestPath to .recover: only for agent service.
recoveryInfo = {}
beforeKillTimeHuman = strftime("%Y-%m-%d %H:%M:%S",
localtime())
beforeKillTimeStamp = int(time.time())
recoveryInfo['beforeKillTimeHuman'] = str(beforeKillTimeHuman)
recoveryInfo['beforeKillTimeStamp'] = str(beforeKillTimeStamp)
# make sure there is always quote: even if null. easy for json parsing in startup
if self._manifest:
recoveryInfo['newManifest'] = self._manifest
else:
recoveryInfo['newManifest'] = 'null'
if oldManifest:
recoveryInfo['oldManifest'] = oldManifest
else:
recoveryInfo['oldManifest'] = 'null'
manifestutil.writeJson('agent', '.recovery', recoveryInfo)
# END Jeff 09 19 2012: save the timestamp and .recover files with
# persist threads result before shutting down
if (hasattr(appGlobal, 'threadMgr')
and appGlobal.threadMgr != None):
killStatus = {}
killStatus['httpStatus'] = 500
killStatus['error'] = Errors.THREAD_KILLED_AGENT_RESTART
killStatus['errorMsg'] = 'thread killed, agent restart'
appGlobal.threadMgr.snapshot(killStatus, True)
self.__activateManifest(self._service, self._manifest)
except SystemExit as exc:
failed = True
if (len(exc.args) == 2):
# ok we got {err code, err msg}
errorCode = exc.args[0]
errorMsg = exc.args[1]
raise exc
except AgentException as exc:
failed = True
errorMsg = 'Activate Manifest - Agent Exception - %s' % exc.getMsg(
)
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = 'Activate Manifest - Unknown error - (%s/%s) - %s - %s' \
% (self._service, self._manifest, str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
if not self._skipCleanupOnFailure():
self.__cleanup(symlinkSwitched, errorMsg, errorCode)
self.__LOG.warning(errorMsg)
self._updateStatus(httpStatus=500,
error=errorCode,
errorMsg=errorMsg)
else:
self._updateProgress(100)
def testDelete(self):
def deleteTestDir(service):
LOG.debug('************ service = %s' % service)
response = self.app.delete(url(controller='service',
service=service,
action='delete'),
expect_errors=True)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
# assert response.status_int == 500, "HTTP response != 500"
def makeTestDir(path):
os.makedirs(path)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'installed-packages'))
def createManifests(mf_path):
os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
latest = os.path.join(mf_path, 'm3.0')
os.makedirs(os.path.join(latest, 'dummy_dir3.0'))
utils.symlink(latest, os.path.join(mf_path, 'active'))
return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')
def makePackageContent(path, pkgPath, pkgPropPath):
pkgFile = file(pkgPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
pkgFile = file(pkgPropPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(path, uname)
def createTestThread(serviceName):
appGlobal = config['pylons.app_globals']
testTh = WaitThread(appGlobal.threadMgr,
ServiceController.serviceCat(serviceName))
testTh.start()
return testTh
def startTestProcess():
cmd = utils.sudoCmd(["sleep", "5"],
pylons.config['app_user_account'])
return Popen(cmd)
if os.name == 'nt':
LOG.warning(
'Services cleanup not supported on windows. Skipping test...')
return
path1 = os.path.join(pylons.config['agent_root'], 'service_nodes',
'foo')
path2 = os.path.join(pylons.config['agent_root'], 'service_nodes',
'bar')
path3 = os.path.join(pylons.config['agent_root'], 'service_nodes',
'agent')
deleteTestDir('foo')
deleteTestDir('bar')
deleteTestDir('agent')
# make dirs
makeTestDir(path1)
makeTestDir(path2)
makeTestDir(path3)
all_mf, active_mf = createManifests(
ServiceController.manifestPath('agent'))
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(ServiceController.serviceRootPath(),
uname)
pkgDir = PackageMgr.packagePath()
pkgPath = os.path.join(pkgDir, "foo.cronus")
pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop")
makePackageContent(pkgDir, pkgPath, pkgPropPath)
# create threads
testThFoo = createTestThread('foo')
testThBar = createTestThread('bar')
testThAgent = createTestThread('agent')
# start process
process = startTestProcess()
# start testing
LOG.debug('************ start cleanup')
response = self.app.post(url(controller='cleanup', action='post'))
LOG.debug('Delete response body = ' + response.body)
body = json.loads(response.body)
tm = time.time()
while (tm + 10 > time.time()):
response = self.app.get(body['status'], expect_errors=True)
LOG.debug('Status response body = ' + response.body)
body = json.loads(response.body)
print body
if (body['progress'] == 100):
break
time.sleep(0.1)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
time.sleep(0.1)
assert not os.path.exists(
path1), 'service foo does exist or is not a directory'
assert not os.path.exists(
path2), 'service bar does exist or is not a directory'
assert os.path.exists(
path3), 'service agent does NOT exist or is not a directory'
assert not testThFoo.isAlive(), 'thread Foo is still alive'
assert not testThBar.isAlive(), 'thread Bar is still alive'
assert not testThAgent.isAlive(), 'thread Agent is still alive'
assert not os.path.exists(pkgPath), 'package foo exists'
assert not os.path.exists(pkgPropPath), 'package prop foo exists'
assert os.path.exists(pkgDir), 'package directory does not exist'
# ensure agent cleanup is proper
active_mf_path = ManifestController.manifestPath('agent', active_mf)
active_link = os.path.join(ServiceController.manifestPath('agent'),
'active')
all_mf.remove(active_mf)
actual_active_mf_path = utils.readlink(active_link)
self.assertTrue(
os.path.exists(active_mf_path),
'active agent manifest got deleted but shouldn\t have')
self.assertTrue(os.path.exists(active_link),
'agent active link missing')
self.assertEqual(
active_mf_path, actual_active_mf_path,
'agent active link pointing to some wrong manifest; link broken?')
for mf in all_mf:
agnt_mf_path = ManifestController.manifestPath('agent', mf)
self.assertFalse(
os.path.exists(agnt_mf_path),
'non active agent mf %s should have been deleted' % mf)
def doRun(self):
""" Main body of the thread """
errorMsg = ""
errorCode = None
symlinkSwitched = False
failed = False
try:
activePath = manifestutil.manifestPath(self._service)
oldManifest = None
# make sure that if the active path exists, it's a link
# if not log that and delete the link
if (os.path.exists(activePath) and not islink(activePath)):
self._LOG.error('%s is not a link. Attempted to delete' % activePath)
shutil.rmtree(activePath)
if (os.path.exists(activePath)):
oldManifest = os.path.basename(readlink(activePath))
# reset requires a valid active link exist
else:
if self.__action == ActivateManifest.ACTION_RESET:
raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
errorMsg = ('No active manifest - cannot %s service'
% self.__action))
if self.__action == ActivateManifest.ACTION_ACTIVATION:
# install new manifest
self.__installManifest(self._service, self._manifest)
self._deactivateManifest(self._service, oldManifest, 11, 50)
if self.__action == ActivateManifest.ACTION_ACTIVATION:
# special logic to handle agent upgrade
# agent upgrade will not run full activation agent will shutdown before activate exit
if self._service == 'agent':
appGlobal = pylons.config['pylons.app_globals']
# persist threads result before shutting down
if (hasattr(appGlobal, 'threadMgr') and appGlobal.threadMgr != None):
killStatus = {}
killStatus['httpStatus'] = 500
killStatus['error'] = Errors.THREAD_KILLED_AGENT_RESTART
killStatus['errorMsg'] = 'thread killed, agent restart'
appGlobal.threadMgr.snapshot(killStatus, True)
# switch active symlink
symlinkSwitched = self.__switchSymlink(self._service, self._manifest)
# activate new manifest
self._activateManifest(self._service, self._manifest, 51, 80)
elif self.__action == ActivateManifest.ACTION_DEACTIVATION:
# remove active link on deactivate
activePath = self.__getSymlinkPath(self._service)
self.__removeSymlink(activePath)
else:
# activate new manifest
self._activateManifest(self._service, self._manifest, 51, 80)
except SystemExit as exc:
failed = True
if (len(exc.args) == 2):
# ok we got {err code, err msg}
errorCode = exc.args[0]
errorMsg = exc.args[1]
raise exc
except AgentException as exc:
failed = True
errorMsg = '%s manifest - Agent Exception - %s' % (self.__action, exc.getMsg())
errorCode = exc.getCode()
except Exception as exc:
failed = True
errorMsg = '%s manifest - Unknown error - (%s/%s) - %s - %s' \
% (self.__action, self._service, self._manifest,
str(exc), traceback.format_exc(5))
errorCode = Errors.UNKNOWN_ERROR
finally:
if failed:
if not self._skipCleanupOnFailure():
self.__cleanup(symlinkSwitched, errorMsg, errorCode)
self._LOG.warning(errorMsg)
self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
else:
self._updateProgress(100)
def testDelete(self):
def deleteTestDir(service):
LOG.debug('************ service = %s' % service)
response = self.app.delete(url(controller='service', service=service, action='delete'), expect_errors = True)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
# assert response.status_int == 500, "HTTP response != 500"
def makeTestDir(path):
os.makedirs(path)
os.makedirs(os.path.join(path, 'manifests'))
os.makedirs(os.path.join(path, 'installed-packages'))
def createManifests(mf_path):
os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
latest = os.path.join(mf_path, 'm3.0')
os.makedirs(os.path.join(latest, 'dummy_dir3.0'))
utils.symlink(latest, os.path.join(mf_path, 'active'))
return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')
def makePackageContent(path, pkgPath, pkgPropPath):
pkgFile = file(pkgPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
pkgFile = file(pkgPropPath, 'w')
for index in range(10):
pkgFile.write(('%s%s') % (index, index))
pkgFile.close()
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(path, uname)
def createTestThread(serviceName):
appGlobal = config['pylons.app_globals']
testTh = WaitThread(appGlobal.threadMgr, ServiceController.serviceCat(serviceName))
testTh.start()
return testTh
def startTestProcess():
cmd = utils.sudoCmd(["sleep", "5"], pylons.config['app_user_account'])
return Popen(cmd)
path1 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'foo')
path2 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'bar')
path3 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'agent')
deleteTestDir('foo')
deleteTestDir('bar')
deleteTestDir('agent')
# make dirs
makeTestDir(path1)
makeTestDir(path2)
makeTestDir(path3)
all_mf, active_mf = createManifests(ServiceController.manifestPath('agent'))
uname = pylons.config['agent_user_account']
TestCleanupController.rchown(ServiceController.serviceRootPath(), uname)
pkgDir = PackageMgr.packagePath()
pkgPath = os.path.join(pkgDir, "foo.cronus")
pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop")
makePackageContent(pkgDir, pkgPath, pkgPropPath)
# create threads
testThFoo = createTestThread('foo')
testThBar = createTestThread('bar')
testThAgent = createTestThread('agent')
# start process
process = startTestProcess()
# start testing
LOG.debug('************ start cleanup')
response = self.app.post(url(controller='cleanup', action='post'))
LOG.debug ('Delete response body = ' + response.body)
body = json.loads(response.body)
tm = time.time()
while (tm + 10 > time.time()):
response = self.app.get(body['status'], expect_errors = True)
LOG.debug ('Status response body = ' + response.body)
body = json.loads(response.body)
print body
if (body['progress'] == 100):
break
time.sleep(0.1)
# make sure the responses are correct
LOG.debug('status = ' + str(response.status_int))
assert response.status_int == 200, "HTTP response != 200"
time.sleep(0.1)
assert not os.path.exists(path1), 'service foo does exist or is not a directory'
assert not os.path.exists(path2), 'service bar does exist or is not a directory'
assert os.path.exists(path3), 'service agent does NOT exist or is not a directory'
assert not testThFoo.isAlive(), 'thread Foo is still alive'
assert not testThBar.isAlive(), 'thread Bar is still alive'
assert not testThAgent.isAlive(), 'thread Agent is still alive'
assert not os.path.exists(pkgPath), 'package foo exists'
assert not os.path.exists(pkgPropPath), 'package prop foo exists'
assert os.path.exists(pkgDir), 'package directory does not exist'
# ensure agent cleanup is proper
active_mf_path = manifestutil.manifestPath('agent', active_mf)
active_link = os.path.join(ServiceController.manifestPath('agent'), 'active')
all_mf.remove(active_mf)
actual_active_mf_path = utils.readlink(active_link)
self.assertTrue(os.path.exists(active_mf_path), 'active agent manifest got deleted but shouldn\t have')
self.assertTrue(os.path.exists(active_link), 'agent active link missing')
self.assertEqual(active_mf_path, actual_active_mf_path, 'agent active link pointing to some wrong manifest; link broken?')
for mf in all_mf:
agnt_mf_path = manifestutil.manifestPath('agent', mf)
self.assertFalse(os.path.exists(agnt_mf_path), 'non active agent mf %s should have been deleted' % mf)