def doRun(self):
        """ Main body of the thread """
        errorMsg = ""
        errorCode = None
        failed = False
        activeManifest = None

        try:
            activePath = manifestutil.manifestPath(self._service, 'active')
            # make sure that the active path exists and it is a link
            # Should we check this again since we already have a check in action controller
            if not os.path.exists(activePath) or not islink(activePath):
                raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, errorMsg = 'No active manifest - cannot restart service')

            activeManifest = os.path.basename(readlink(activePath))

            self.__lcmActionManifest(self._service, activeManifest, self.__action)

            self._LOG.info('Done: %s service for (%s/%s)' % (self.__action, self._service, activeManifest))
            self._updateStatus(progress = 100)

        except AgentException as exc:
            failed = True
            errorMsg = '%s Service - Agent Exception - %s' % (self.__action, exc.getMsg())
            errorCode = exc.getCode()
        except Exception as exc:
            failed = True
            errorMsg = '%s Service - Unknown error - (%s/%s) - %s - %s' \
                        % (self.__action, self._service, self._manifest, str(exc), traceback.format_exc(5))
            errorCode = Errors.UNKNOWN_ERROR
        finally:
            if failed:
                self._LOG.error(errorMsg)

                self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
Esempio n. 2
0
    def get(self, module):
        """ Get a new service object """
        try:
            # make sure the service path exists
            path = manifestutil.modulePath("agent", module)
            if not os.path.exists(path):
                return errorResult(
                    request,
                    response,
                    error=Errors.SERVICE_NOT_FOUND,
                    errorMsg="Unable to find module (%s)" % module,
                    controller=self,
                )

            result = {}
            modulePackage = readlink(path)
            result["package"] = modulePackage

            return doneResult(request, response, result=result, controller=self)

        except Exception as excep:
            return errorResult(
                request,
                response,
                error=Errors.UNKNOWN_ERROR,
                errorMsg="Unknown error when getting module (%s) %s - %s"
                % (module, str(excep), traceback.format_exc(2)),
                controller=self,
            )
Esempio n. 3
0
    def getAllSymLinks(service):
        """ return all the symlinks from manifests to packages for a given service"""

        LOG.debug('calling getAllSymLinks %s' % service)
        linkedPaths = []
        manPath = ServiceController.manifestPath(service)
        LOG.debug('manifestPath is %s' % manPath)
        for path in os.listdir(manPath):
            LOG.debug('path is %s' % path)
            if not islink(os.path.join(manPath, path)):
                LOG.debug('path is dir not a link')
                pkgPaths = [
                    packageDir
                    for packageDir in os.listdir(os.path.join(manPath, path))
                ]
                LOG.debug('pkgPaths is %s' % pkgPaths)
                for pkgPath in pkgPaths:
                    try:
                        LOG.debug('pkgPath is %s' % pkgPath)
                        if not os.path.isfile(
                                os.path.join(
                                    manPath, path, pkgPath)) and islink(
                                        os.path.join(manPath, path, pkgPath)):
                            LOG.debug('pkgPaths is %s' % pkgPath)
                            targetPath = os.path.abspath(
                                readlink(os.path.join(manPath, path, pkgPath)))
                            linkedPaths.append(targetPath)
                            LOG.debug('targetPath is %s ' % targetPath)
                    except BaseException as exc:
                        LOG.error('failed to read link for the pkg path %s' %
                                  str(exc))
        return linkedPaths
Esempio n. 4
0
    def delete(self, service, manifest):
        """ Delete a new service object """
        try:
            path = manifestutil.manifestPath(service, manifest)
            if (not os.path.isdir(path)):
                return errorResult(request, response, Errors.MANIFEST_NOT_FOUND,
                                   'manifest (%s/%s) missing service' % (service, manifest),
                                   controller=self)

            # first check that this isn't the active manifest
            path = manifestutil.manifestPath(service)
            if (os.path.exists(path)):
                activePath = os.path.basename(readlink(path))
                deletePath = os.path.basename(manifestutil.manifestPath(service, manifest))

                if (activePath == deletePath):
                    return errorResult(request, response, Errors.MANIFEST_DELETING_ACTIVE_MANIFEST,
                                       'Manifest(%s, %s) attempting to delete active manifest'
                                       % (service, manifest),
                                       controller=self)

            # now try to delete the manifest directory
            appGlobal = config['pylons.app_globals']
            manThread = ManifestDelete(appGlobal.threadMgr, service, manifest)
            self.injectJobCtx(manThread)
            manThread.start()
            manThread.threadMgrEvent.wait()

            return statusResult(request, response, manThread, controller=self)

        except Exception as excep:
            return errorResult(request, response, error=Errors.UNKNOWN_ERROR,
                               errorMsg='Unknown error for delete manifest(%s/%s) - %s - %s' % 
                               (service, manifest, str(excep), traceback.format_exc(2)),
                               controller=self)
Esempio n. 5
0
    def get(self, module):
        """ Get a new service object """
        try:
            # make sure the service path exists
            path = manifestutil.modulePath('agent', module)
            if (not os.path.exists(path)):
                return errorResult(request,
                                   response,
                                   error=Errors.SERVICE_NOT_FOUND,
                                   errorMsg='Unable to find module (%s)' %
                                   module,
                                   controller=self)

            result = {}
            modulePackage = readlink(path)
            result['package'] = modulePackage

            return doneResult(request,
                              response,
                              result=result,
                              controller=self)

        except Exception as excep:
            return errorResult(
                request,
                response,
                error=Errors.UNKNOWN_ERROR,
                errorMsg='Unknown error when getting module (%s) %s - %s' %
                (module, str(excep), traceback.format_exc(2)),
                controller=self)
Esempio n. 6
0
def activateManifest(testController, manifest = 'bar', service = 'foo'):

    body = json.dumps({'manifest':manifest})

    response = testController.app.post(url(controller = 'manifest', action = 'activate', service = service, manifest = manifest),
                                       headers = {'Content-Type' : 'application/json'},
                                       params = body)

    assert response.status_int == 200, 'Action get assert'

    body = json.loads(response.body)

    tm = time.time()
    while (tm + 120 > time.time()):
        response = testController.app.get(body['status'])
        body = json.loads(response.body)

        LOG.debug("activateManifest ********** progress = %s" % body['progress'])
        if (int(body['progress']) == 100):
            break
        time.sleep(0.1)

    LOG.debug('status = ' + str(response.status_int))
    assert response.status_int == 200, "HTTP response != 200"
    LOG.debug ('Status response body = %s' % str(body))
    assert body['progress'] == 100

    # let's make sure the link is there correctly
    activePath = os.path.join(ServiceController.manifestPath(service), 'active')
    LOG.debug ('active path = ' + activePath)
    assert islink(activePath)

    link = readlink(activePath)
    LOG.debug ('link = ' + link)
    assert link == manifest
Esempio n. 7
0
def activeManifestPath(service):
    """ compute the path to this __service active manifest """
    activePath = manifestPath(service, ACTIVE_MANIFEST)
    if (not os.path.exists(activePath)):
        return ''

    realManifest = readlink(activePath)
    return realManifest
Esempio n. 8
0
def activeManifestPath(service):
    """ compute the path to this __service active manifest """
    activePath = manifestPath(service, ACTIVE_MANIFEST)
    if (not os.path.exists(activePath)):
        return ''

    realManifest = readlink(activePath)
    return realManifest
    def doRun(self):
        """ Main body of the thread """
        errorMsg = ""
        errorCode = None
        failed = False
        try:

            activePath = os.path.join(
                ServiceController.manifestPath(self._service), 'active')
            oldManifest = None

            # make sure that if the active path exists, it's a link
            # if not log that and delete the link
            if (os.path.exists(activePath) and not os.name == 'nt'
                    and not islink(activePath)):
                self.__LOG.error('%s is not a link.  Attempted to delete' %
                                 activePath)
                shutil.rmtree(activePath)

            if (os.path.exists(activePath)):
                oldManifest = os.path.basename(readlink(activePath))
            else:
                raise AgentException(
                    error=Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
                    errorMsg='No active manifest - cannot deactivate service')

            self.__deactivateManifest(self._service, oldManifest)
            self.__removeSymlink(self._service)

        except SystemExit as exc:
            failed = True
            if (len(exc.args) == 2):
                # ok we got {err code, err msg}
                errorCode = exc.args[0]
                errorMsg = exc.args[1]
            raise exc
        except AgentException as exc:
            failed = True
            errorMsg = 'Deactivate Manifest - Agent Exception - %s' % exc.getMsg(
            )
            errorCode = exc.getCode()
        except Exception as exc:
            failed = True
            errorMsg = 'Deactivate Manifest - Unknown error - (%s) - %s - %s' \
                        % (self._service, str(exc), traceback.format_exc(5))
            errorCode = Errors.UNKNOWN_ERROR
        finally:
            if failed:
                self.__LOG.warning(errorMsg)
                self._updateStatus(httpStatus=500,
                                   error=errorCode,
                                   errorMsg=errorMsg)
            self.__LOG.debug('Done: activate manifest for (%s)' %
                             (self._service))
            self._updateProgress(100)
Esempio n. 10
0
    def getActiveManifestPath(service):
        """
        return the name of the active manifest under a specific service
        @param service: name of service
        @return: path of active manifest, or empty string if no active manifest
        """
        activePath = os.path.join(ServiceController.manifestPath(service), 'active')
        if (not os.path.exists(activePath)):
            return ''

        return readlink(activePath)
Esempio n. 11
0
    def doRun(self):
        """ Main body of the thread """
        errorMsg = ""
        errorCode = None
        failed = False
        activeManifest = None

        try:
            activePath = manifestutil.manifestPath(self._service, 'active')
            
            # make sure that the active path exists and it is a link
            # Should we check this again since we already have a check in action controller
            if not os.path.exists(activePath) or not islink(activePath):
                raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, 
                                     errorMsg = 'Service %s has no active manifest' % self._service)

            activeManifest = os.path.basename(readlink(activePath))

            if self.__action == StartStopService.ACTION_SHUTDOWN:
                self._shutdownManifest(self._service, activeManifest, 50, 90)
            elif self.__action == StartStopService.ACTION_STARTUP:
                self._startupManifest(self._service, activeManifest, 50, 90)
            elif self.__action == StartStopService.ACTION_RESTART:
                self._restartManifest(self._service, activeManifest, 10, 90)
            else:
                raise AgentException(error = Errors.INVALID_LIFECYCLE_ACTION, 
                                     errorMsg = 'Invalid life cycle action - %s' % self.__action)

            self._LOG.info('Done: %s service for (%s/%s)' % (self.__action, self._service, activeManifest))
            self._updateStatus(progress = 100)

        except AgentException as exc:
            failed = True
            errorMsg = '%s Service - Agent Exception - %s' % (self.__action, exc.getMsg())
            errorCode = exc.getCode()
            
        except Exception as exc:
            failed = True
            errorMsg = '%s Service - Unknown error - (%s/%s) - %s - %s' \
                        % (self.__action, self._service, self._manifest, str(exc), traceback.format_exc(5))
            errorCode = Errors.UNKNOWN_ERROR
            
        finally:
            if failed:
                self._LOG.error(errorMsg)

                if not self._skipCleanupOnFailure() and self.__action != StartStopService.ACTION_SHUTDOWN and self._service and activeManifest:
                    try:
                        self._LOG.info('%s Service %s failed, shutdown to cleanup' % (self.__action, self._service))
                        self._shutdownManifest(self._service, activeManifest, 91, 99)
                    except BaseException as excep:
                        self._LOG.error('Cleanup failed - %s' % str(excep))

                self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
Esempio n. 12
0
    def getActiveManifestPath(service):
        """
        return the name of the active manifest under a specific service
        @param service: name of service
        @return: path of active manifest, or empty string if no active manifest
        """
        activePath = os.path.join(ServiceController.manifestPath(service),
                                  'active')
        if (not os.path.exists(activePath)):
            return ''

        return readlink(activePath)
Esempio n. 13
0
    def doRun(self):
        """ Main body of the thread """
        errorMsg = ""
        errorCode = None
        failed = False
        activeManifest = None
        try:
            activePath = os.path.join(
                ServiceController.manifestPath(self._service), 'active')
            # make sure that the active path exists and it is a link
            # Should we check this again since we already have a check in action controller
            if not os.path.exists(activePath) or not islink(activePath):
                raise AgentException(
                    error=Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
                    errorMsg='No active manifest - cannot reset service')

            activeManifest = os.path.basename(readlink(activePath))

            self.__shutdownManifest(self._service, activeManifest)
            self.__deactivateManifest(self._service, activeManifest)
            self.__activateManifest(self._service, activeManifest)
            self.__startupManifest(self._service, activeManifest)

            self.__LOG.info('Done: reset service for (%s/%s)' %
                            (self._service, activeManifest))
            self._updateStatus(progress=100)

        except AgentException as exc:
            failed = True
            errorMsg = 'Activate Manifest - Agent Exception - %s' % exc.getMsg(
            )
            errorCode = exc.getCode()
        except Exception as exc:
            failed = True
            errorMsg = 'Activate Manifest - Unknown error - (%s/%s) - %s - %s' \
                        % (self._service, self._manifest, str(exc), traceback.format_exc(5))
            errorCode = Errors.UNKNOWN_ERROR
        finally:
            if failed:
                self.__LOG.error(errorMsg)

                if self._service and activeManifest:
                    try:
                        self.__LOG.info(
                            'Reset service %s failed, shutdown to cleanup' %
                            self._service)
                        self.__shutdownManifest(self._service, activeManifest)
                    except BaseException as excep:
                        self.__LOG.error('Cleanup failed - %s' % str(excep))

                self._updateStatus(httpStatus=500,
                                   error=errorCode,
                                   errorMsg=errorMsg)
Esempio n. 14
0
    def get(self, service):
        """ Get a new service object """
        try:
            from agent.lib.agent_thread.manifest_create import ManifestCreate

            # make sure the service path exists
            path = ServiceController.servicePath(service)
            if (not os.path.exists(path)):
                return errorResult(request,
                                   response,
                                   error=Errors.SERVICE_NOT_FOUND,
                                   errorMsg='Unable to find service (%s)' %
                                   service,
                                   controller=self)

            path = ServiceController.manifestPath(service)

            activeManifest = None
            manifestList = []
            for manifest in os.listdir(path):
                if (ManifestCreate.isInProgress(manifest)):
                    continue

                manifestPath = os.path.join(path, manifest)
                if (manifest == 'active'):
                    activeLink = readlink(manifestPath)

                    if (activeLink == None):
                        manifestList.append(manifest)
                    else:
                        activeManifest = os.path.basename(activeLink)
                else:
                    manifestList.append(manifest)

            result = {}
            manifestList.sort()
            result['manifest'] = manifestList
            result['activemanifest'] = activeManifest

            return doneResult(request,
                              response,
                              result=result,
                              controller=self)

        except Exception as excep:
            return errorResult(
                request,
                response,
                error=Errors.UNKNOWN_ERROR,
                errorMsg='Unknown error when getting service (%s) %s - %s' %
                (service, str(excep), traceback.format_exc(2)),
                controller=self)
Esempio n. 15
0
    def get(self, service):
        """ Get a new service object """
        try:
            from agent.lib.agent_thread.manifest_create import ManifestCreate

            # make sure the service path exists
            path = ServiceController.servicePath(service)
            if not os.path.exists(path):
                return errorResult(
                    request,
                    response,
                    error=Errors.SERVICE_NOT_FOUND,
                    errorMsg="Unable to find service (%s)" % service,
                    controller=self,
                )

            path = ServiceController.manifestPath(service)

            activeManifest = None
            manifestList = []
            for manifest in os.listdir(path):
                if ManifestCreate.isInProgress(manifest):
                    continue

                manifestPath = os.path.join(path, manifest)
                if manifest == "active":
                    activeLink = readlink(manifestPath)

                    if activeLink == None:
                        manifestList.append(manifest)
                    else:
                        activeManifest = os.path.basename(activeLink)
                else:
                    manifestList.append(manifest)

            result = {}
            manifestList.sort()
            result["manifest"] = manifestList
            result["activemanifest"] = activeManifest

            return doneResult(request, response, result=result, controller=self)

        except Exception as excep:
            return errorResult(
                request,
                response,
                error=Errors.UNKNOWN_ERROR,
                errorMsg="Unknown error when getting service (%s) %s - %s"
                % (service, str(excep), traceback.format_exc(2)),
                controller=self,
            )
Esempio n. 16
0
    def doRun(self):
        """ Main body of the thread """
        errorMsg = ""
        errorCode = None
        failed = False
        activeManifest = None

        try:
            activePath = manifestutil.manifestPath(self._service, 'active')
            # make sure that the active path exists and it is a link
            # Should we check this again since we already have a check in action controller
            if not os.path.exists(activePath) or not islink(activePath):
                raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, errorMsg = 'No active manifest - cannot restart service')

            activeManifest = os.path.basename(readlink(activePath))

            if self.__action == StartStopService.ACTION_SHUTDOWN:
                self.__shutdownManifest(self._service, activeManifest)
            elif self.__action == StartStopService.ACTION_STARTUP:
                self.__startupManifest(self._service, activeManifest)
            elif self.__action == StartStopService.ACTION_RESTART:
                self.__restartManifest(self._service, activeManifest)
            elif self.__action == StartStopService.ACTION_REBOOT:
                self.__rebootManifest(self._service, activeManifest)
            else:
                raise AgentException(error = Errors.INVALID_LIFECYCLE_ACTION, errorMsg = 'Invalid life cycle action - %s' % self.__action)

            self.__LOG.info('Done: %s service for (%s/%s)' % (self.__action, self._service, activeManifest))
            self._updateStatus(progress = 100)

        except AgentException as exc:
            failed = True
            errorMsg = '%s Service - Agent Exception - %s' % (self.__action, exc.getMsg())
            errorCode = exc.getCode()
        except Exception as exc:
            failed = True
            errorMsg = '%s Service - Unknown error - (%s/%s) - %s - %s' \
                        % (self.__action, self._service, self._manifest, str(exc), traceback.format_exc(5))
            errorCode = Errors.UNKNOWN_ERROR
        finally:
            if failed:
                self.__LOG.error(errorMsg)

                if not self._skipCleanupOnFailure() and self.__action != StartStopService.ACTION_SHUTDOWN and self._service and activeManifest:
                    try:
                        self.__LOG.info('%s Service %s failed, shutdown to cleanup' % (self.__action, self._service))
                        self.__shutdownManifest(self._service, activeManifest)
                    except BaseException as excep:
                        self.__LOG.error('Cleanup failed - %s' % str(excep))

                self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
Esempio n. 17
0
 def getLocalPyPkg():
     """
     reuse of local python package instead of download it again from source of truth,
     this should be the common use case for selfupdate without needing to update the python package
     """
     activeManifest = manifestutil.getActiveManifest('agent')
     activePyLink = os.path.join(manifestutil.manifestPath('agent', activeManifest), 'python_package')
     if (os.path.exists(activePyLink)):
         activePyPath = readlink(activePyLink)
         pyVersion = os.path.basename(activePyPath)
         pyPkgName = ('python_package-%s' % pyVersion)
         if AgentUpdate.nameRe.match(pyPkgName):
             return ('http://localhost:12020/%s.cronus' % pyPkgName)
         else: 
             raise AgentException(Errors.PACKAGE_SCHEME_ERROR, 'package name %s is not valid' % pyPkgName)
Esempio n. 18
0
    def doRun(self):
        """ Main body of the thread """
        errorMsg = ""
        errorCode = None
        failed = False
        try:

            activePath = os.path.join(ServiceController.manifestPath(self._service), 'active')
            oldManifest = None

            # make sure that if the active path exists, it's a link
            # if not log that and delete the link
            if (os.path.exists(activePath) and not os.name == 'nt' and not islink(activePath)):
                self.__LOG.error('%s is not a link.  Attempted to delete' % activePath)
                shutil.rmtree(activePath)

            if (os.path.exists(activePath)):
                oldManifest = os.path.basename(readlink(activePath))
            else:
                raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, errorMsg = 'No active manifest - cannot deactivate service')

            self.__deactivateManifest(self._service, oldManifest)
            self.__removeSymlink(self._service)

        except SystemExit as exc:
            failed = True
            if (len(exc.args) == 2):
                # ok we got {err code, err msg}
                errorCode = exc.args[0]
                errorMsg = exc.args[1]
            raise exc
        except AgentException as exc:
            failed = True
            errorMsg = 'Deactivate Manifest - Agent Exception - %s' % exc.getMsg()
            errorCode = exc.getCode()
        except Exception as exc:
            failed = True
            errorMsg = 'Deactivate Manifest - Unknown error - (%s) - %s - %s' \
                        % (self._service, str(exc), traceback.format_exc(5))
            errorCode = Errors.UNKNOWN_ERROR
        finally:
            if failed:
                self.__LOG.warning(errorMsg)
                self._updateStatus(httpStatus = 500, error = errorCode,
                                   errorMsg = errorMsg)
            self.__LOG.debug('Done: activate manifest for (%s)' % (self._service))
            self._updateProgress(100)
Esempio n. 19
0
def getModuleSymLinks(service = 'agent'):
    """ return all the symlinks from manifests to packages for a given service"""

    LOG.debug('calling getAllSymLinks %s' % service)
    linkedPaths = []
    mRootPath = moduleRootPath(service)
    LOG.debug('moduleRootPath is %s' % mRootPath)
    for pkgPath in os.listdir(mRootPath):
        try:
            LOG.debug('pkgPath is %s' % pkgPath)
            if not os.path.isfile(os.path.join(mRootPath, pkgPath)) and islink(os.path.join(mRootPath, pkgPath)):
                targetPath = os.path.abspath(readlink(os.path.join(mRootPath, pkgPath)))
                linkedPaths.append(targetPath)
                LOG.debug('targetPath is %s ' % targetPath)
        except BaseException as exc:
            LOG.error('failed to read link for the pkg path %s' % str(exc))
    return linkedPaths
Esempio n. 20
0
def getModuleSymLinks(service = 'agent'):
    """ return all the symlinks from manifests to packages for a given service"""

    LOG.debug('calling getAllSymLinks %s' % service)
    linkedPaths = []
    mRootPath = moduleRootPath(service)
    LOG.debug('moduleRootPath is %s' % mRootPath)
    for pkgPath in os.listdir(mRootPath):
        try:
            LOG.debug('pkgPath is %s' % pkgPath)
            if not os.path.isfile(os.path.join(mRootPath, pkgPath)) and islink(os.path.join(mRootPath, pkgPath)):
                targetPath = os.path.abspath(readlink(os.path.join(mRootPath, pkgPath)))
                linkedPaths.append(targetPath)
                LOG.debug('targetPath is %s ' % targetPath)
        except BaseException as exc:
            LOG.error('failed to read link for the pkg path %s' % str(exc))
    return linkedPaths
Esempio n. 21
0
    def delete(self, service, manifest):
        """ Delete a new service object """
        try:
            path = ManifestController.manifestPath(service, manifest)
            if (not os.path.isdir(path)):
                return errorResult(request,
                                   response,
                                   Errors.MANIFEST_NOT_FOUND,
                                   'manifest (%s/%s) missing service' %
                                   (service, manifest),
                                   controller=self)

            # first check that this isn't the active manifest
            path = os.path.join(ServiceController.manifestPath(service),
                                'active')
            if (os.path.exists(path)):
                activePath = os.path.basename(readlink(path))
                deletePath = os.path.basename(
                    ManifestController.manifestPath(service, manifest))

                if (activePath == deletePath):
                    return errorResult(
                        request,
                        response,
                        Errors.MANIFEST_DELETING_ACTIVE_MANIFEST,
                        'Manifest(%s, %s) attempting to delete active manifest'
                        % (service, manifest),
                        controller=self)

            # now try to delete the manifest directory
            appGlobal = config['pylons.app_globals']
            manThread = ManifestDelete(appGlobal.threadMgr, service, manifest)
            self.injectJobCtx(manThread)
            manThread.start()
            manThread.threadMgrEvent.wait()

            return statusResult(request, response, manThread, controller=self)

        except Exception as excep:
            return errorResult(
                request,
                response,
                error=Errors.UNKNOWN_ERROR,
                errorMsg='Unknown error for delete manifest(%s/%s) - %s - %s' %
                (service, manifest, str(excep), traceback.format_exc(2)),
                controller=self)
Esempio n. 22
0
    def get(self, service, manifest):
        """ Get a new service object """
        LOG.info('Get for service (%s) and manifest (%s)', service, manifest)

        try:
            # first check that the manifest directory exists
            path = ManifestController.manifestPath(service, manifest)
            if (not os.path.isdir(path)):
                return errorResult(request,
                                   response,
                                   Errors.MANIFEST_NOT_FOUND,
                                   'manifest (%s/%s) missing service' %
                                   (service, manifest),
                                   controller=self)

            # now go through the list of packages in the manifest
            packages = []
            packageLinkNames = glob.glob(
                os.path.join(self.manifestPath(service, manifest), '*'))
            for packageLink in packageLinkNames:

                package = readlink(packageLink)

                LOG.debug('Get package (%s) in manifest (%s)', package,
                          manifest)

                # deal with the case where the package has a / or \ (for windoz) at the end
                package = package.rstrip('/')
                package = package.rstrip('\\')

                # the name of the package can be constructed from the last two path components
                (head, version) = os.path.split(package)
                (head, name) = os.path.split(head)

                LOG.debug('Add package %s-%s.cronus' % (name, version))
                packages.append('%s-%s.cronus' % (name, version))

        except OSError as excp:
            return errorResult(request,
                               response,
                               Errors.MANIFEST_PATH_ERROR,
                               'Manifest(%s, %s) path error: %s' %
                               (service, manifest, str(excp)),
                               controller=self)

        return doneResult(request, response, result=packages, controller=self)
Esempio n. 23
0
    def doRun(self):
        """ Main body of the thread """
        errorMsg = ""
        errorCode = None
        failed = False
        activeManifest = None
        try:
            activePath = os.path.join(ServiceController.manifestPath(self._service), 'active')
            # make sure that the active path exists and it is a link
            # Should we check this again since we already have a check in action controller
            if not os.path.exists(activePath) or not islink(activePath):
                raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, errorMsg = 'No active manifest - cannot reset service')

            activeManifest = os.path.basename(readlink(activePath))

            self.__shutdownManifest(self._service, activeManifest)
            self.__deactivateManifest(self._service, activeManifest)
            self.__activateManifest(self._service, activeManifest)
            self.__startupManifest(self._service, activeManifest)

            self.__LOG.info('Done: reset service for (%s/%s)' % (self._service, activeManifest))
            self._updateStatus(progress = 100)

        except AgentException as exc:
            failed = True
            errorMsg = 'Activate Manifest - Agent Exception - %s' % exc.getMsg()
            errorCode = exc.getCode()
        except Exception as exc:
            failed = True
            errorMsg = 'Activate Manifest - Unknown error - (%s/%s) - %s - %s' \
                        % (self._service, self._manifest, str(exc), traceback.format_exc(5))
            errorCode = Errors.UNKNOWN_ERROR
        finally:
            if failed:
                self.__LOG.error(errorMsg)

                if self._service and activeManifest:
                    try:
                        self.__LOG.info('Reset service %s failed, shutdown to cleanup' % self._service)
                        self.__shutdownManifest(self._service, activeManifest)
                    except BaseException as excep:
                        self.__LOG.error('Cleanup failed - %s' % str(excep))

                self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
Esempio n. 24
0
 def getLocalPyPkg():
     """
     reuse of local python package instead of download it again from source of truth,
     this should be the common use case for selfupdate without needing to update the python package
     """
     activeManifest = manifestutil.getActiveManifest('agent')
     activePyLink = os.path.join(
         manifestutil.manifestPath('agent', activeManifest),
         'python_package')
     if (os.path.exists(activePyLink)):
         activePyPath = readlink(activePyLink)
         pyVersion = os.path.basename(activePyPath)
         pyPkgName = ('python_package-%s' % pyVersion)
         if AgentUpdate.nameRe.match(pyPkgName):
             return ('http://localhost:12020/%s.cronus' % pyPkgName)
         else:
             raise AgentException(
                 Errors.PACKAGE_SCHEME_ERROR,
                 'package name %s is not valid' % pyPkgName)
Esempio n. 25
0
def activateManifest(testController, manifest='bar', service='foo'):

    body = json.dumps({'manifest': manifest})

    response = testController.app.post(
        url(controller='action', action='activatemanifest', service=service),
        headers={'Content-Type': 'application/json'},
        params=body)

    assert response.status_int == 200, 'Action get assert'

    body = json.loads(response.body)

    tm = time.time()
    while (tm + 120 > time.time()):
        response = testController.app.get(body['status'])
        body = json.loads(response.body)

        LOG.debug("activateManifest ********** progress = %s" %
                  body['progress'])
        if (int(body['progress']) == 100):
            break
        time.sleep(0.1)

    LOG.debug('status = ' + str(response.status_int))
    assert response.status_int == 200, "HTTP response != 200"
    LOG.debug('Status response body = %s' % str(body))
    assert body['progress'] == 100

    # let's make sure the link is there correctly
    activePath = os.path.join(ServiceController.manifestPath(service),
                              'active')
    LOG.debug('active path = ' + activePath)
    assert islink(activePath)

    link = readlink(activePath)
    LOG.debug('link = ' + link)
    if os.name == 'nt':
        manifestPath = os.path.join(ServiceController.manifestPath(service),
                                    manifest)
        assert (link == manifestPath or link == manifest)
    else:
        assert link == manifest
Esempio n. 26
0
    def get(self, service, manifest):
        """ Get a new service object """
        LOG.info('Get for service (%s) and manifest (%s)', service, manifest)

        try:
            # first check that the manifest directory exists
            path = manifestutil.manifestPath(service, manifest)
            if (not os.path.isdir(path)):
                return errorResult(request, response, Errors.MANIFEST_NOT_FOUND,
                                   'manifest (%s/%s) missing service' % (service, manifest),
                                   controller=self)

            # now go through the list of packages in the manifest
            packages = []
            packageLinkNames = glob.glob(os.path.join(manifestutil.manifestPath(service, manifest), '*'))
            for packageLink in packageLinkNames:

                package = readlink(packageLink)

                LOG.debug('Get package (%s) in manifest (%s)', package, manifest)

                # deal with the case where the package has a / or \ (for windoz) at the end
                package = package.rstrip('/')
                package = package.rstrip('\\')

                # the name of the package can be constructed from the last two path components
                (head, version) = os.path.split(package)
                (head, name) = os.path.split(head)

                LOG.debug('Add package %s-%s.cronus' % (name, version))
                packages.append('%s-%s.cronus' % (name, version))

        except OSError as excp:
            return errorResult(request, response, Errors.MANIFEST_PATH_ERROR,
                               'Manifest(%s, %s) path error: %s' % (service, manifest, str(excp)),
                               controller=self)

        return doneResult(request, response, result=packages, controller=self)
Esempio n. 27
0
def getAllSymLinks(service):
    """ return all the symlinks from manifests to packages for a given service"""

    LOG.debug('calling getAllSymLinks %s' % service)
    linkedPaths = []
    manPath = manifestRootPath(service)
    LOG.debug('manifestPath is %s' % manPath)
    for path in os.listdir(manPath):
        LOG.debug('path is %s' % path)
        if not islink(os.path.join(manPath, path)):
            LOG.debug('path is dir not a link')
            pkgPaths = [ packageDir for packageDir in os.listdir(os.path.join(manPath, path)) ]
            LOG.debug('pkgPaths is %s' % pkgPaths)
            for pkgPath in pkgPaths:
                try:
                    LOG.debug('pkgPath is %s' % pkgPath)
                    if not os.path.isfile(os.path.join(manPath, path, pkgPath)) and islink(os.path.join(manPath, path, pkgPath)):
                        LOG.debug('pkgPaths is %s' % pkgPath)
                        targetPath = os.path.abspath(readlink(os.path.join(manPath, path, pkgPath)))
                        linkedPaths.append(targetPath)
                        LOG.debug('targetPath is %s ' % targetPath)
                except BaseException as exc:
                    LOG.error('failed to read link for the pkg path %s' % str(exc))
    return linkedPaths
Esempio n. 28
0
    def doRun(self):
        """ Main body of the thread """
        errorMsg = ""
        errorCode = None
        symlinkSwitched = False
        failed = False
        try:
            activePath = os.path.join(ServiceController.manifestPath(self._service), 'active')
            oldManifest = None
            
            appGlobal = pylons.config['pylons.app_globals']

            # make sure that if the active path exists, it's a link
            # if not log that and delete the link
            if (os.path.exists(activePath) and not os.name == 'nt' and not islink(activePath)):
                self.__LOG.error('%s is not a link.  Attempted to delete' % activePath)
                shutil.rmtree(activePath)

            if (os.path.exists(activePath)):
                oldManifest = os.path.basename(readlink(activePath))

            self.__installManifest(self._service, self._manifest)
            self.__deactivateManifest(self._service, oldManifest)
            symlinkSwitched = self.__switchSymlink(self._service, self._manifest)

            appGlobal = pylons.config['pylons.app_globals']

            if self._service == 'agent':
                # START Jeff  09 19 2012: save the timestamp and old manifestPath to .recover: only for agent service.
                recoveryInfo = {}
                beforeKillTimeHuman = strftime("%Y-%m-%d %H:%M:%S", localtime())
                beforeKillTimeStamp = int(time.time())
                recoveryInfo['beforeKillTimeHuman'] = str(beforeKillTimeHuman) 
                recoveryInfo['beforeKillTimeStamp'] = str(beforeKillTimeStamp) 
                # make sure there is always quote: even if null. easy for json parsing in startup
                if self._manifest:
                    recoveryInfo['newManifest'] = self._manifest
                else:
                    recoveryInfo['newManifest'] = 'null'
                if oldManifest:
                    recoveryInfo['oldManifest'] = oldManifest
                else:
                    recoveryInfo['oldManifest'] = 'null'
                manifestutil.writeJson('agent', '.recovery', recoveryInfo)
                # END Jeff  09 19 2012: save the timestamp and .recover files with

                # persist threads result before shutting down
                if (hasattr(appGlobal, 'threadMgr') and appGlobal.threadMgr != None):
                    killStatus = {}
                    killStatus['httpStatus'] = 500
                    killStatus['error'] = Errors.THREAD_KILLED_AGENT_RESTART
                    killStatus['errorMsg'] = 'thread killed, agent restart'
                    appGlobal.threadMgr.snapshot(killStatus, True)

            self.__activateManifest(self._service, self._manifest)

        except SystemExit as exc:
            failed = True
            if (len(exc.args) == 2):
                # ok we got {err code, err msg}
                errorCode = exc.args[0]
                errorMsg = exc.args[1]
            raise exc
        
        except AgentException as exc:
            failed = True
            errorMsg = 'Activate Manifest - Agent Exception - %s' % exc.getMsg()
            errorCode = exc.getCode()
        
        except Exception as exc:
            failed = True
            errorMsg = 'Activate Manifest - Unknown error - (%s/%s) - %s - %s' \
                        % (self._service, self._manifest, str(exc), traceback.format_exc(5))
            errorCode = Errors.UNKNOWN_ERROR
        
        finally:
            if failed: 
                if not self._skipCleanupOnFailure():
                    self.__cleanup(symlinkSwitched, errorMsg, errorCode)

                self.__LOG.warning(errorMsg)
                self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
            else:
                self._updateProgress(100)
Esempio n. 29
0
    def doRun(self):
        """ Main body of the thread """
        errorMsg = ""
        errorCode = None
        symlinkSwitched = False
        failed = False
        try:
            activePath = os.path.join(
                ServiceController.manifestPath(self._service), 'active')
            oldManifest = None

            appGlobal = pylons.config['pylons.app_globals']

            # make sure that if the active path exists, it's a link
            # if not log that and delete the link
            if (os.path.exists(activePath) and not os.name == 'nt'
                    and not islink(activePath)):
                self.__LOG.error('%s is not a link.  Attempted to delete' %
                                 activePath)
                shutil.rmtree(activePath)

            if (os.path.exists(activePath)):
                oldManifest = os.path.basename(readlink(activePath))

            self.__installManifest(self._service, self._manifest)
            self.__deactivateManifest(self._service, oldManifest)
            symlinkSwitched = self.__switchSymlink(self._service,
                                                   self._manifest)

            appGlobal = pylons.config['pylons.app_globals']

            if self._service == 'agent':
                # START Jeff  09 19 2012: save the timestamp and old manifestPath to .recover: only for agent service.
                recoveryInfo = {}
                beforeKillTimeHuman = strftime("%Y-%m-%d %H:%M:%S",
                                               localtime())
                beforeKillTimeStamp = int(time.time())
                recoveryInfo['beforeKillTimeHuman'] = str(beforeKillTimeHuman)
                recoveryInfo['beforeKillTimeStamp'] = str(beforeKillTimeStamp)
                # make sure there is always quote: even if null. easy for json parsing in startup
                if self._manifest:
                    recoveryInfo['newManifest'] = self._manifest
                else:
                    recoveryInfo['newManifest'] = 'null'
                if oldManifest:
                    recoveryInfo['oldManifest'] = oldManifest
                else:
                    recoveryInfo['oldManifest'] = 'null'
                manifestutil.writeJson('agent', '.recovery', recoveryInfo)
                # END Jeff  09 19 2012: save the timestamp and .recover files with

                # persist threads result before shutting down
                if (hasattr(appGlobal, 'threadMgr')
                        and appGlobal.threadMgr != None):
                    killStatus = {}
                    killStatus['httpStatus'] = 500
                    killStatus['error'] = Errors.THREAD_KILLED_AGENT_RESTART
                    killStatus['errorMsg'] = 'thread killed, agent restart'
                    appGlobal.threadMgr.snapshot(killStatus, True)

            self.__activateManifest(self._service, self._manifest)

        except SystemExit as exc:
            failed = True
            if (len(exc.args) == 2):
                # ok we got {err code, err msg}
                errorCode = exc.args[0]
                errorMsg = exc.args[1]
            raise exc

        except AgentException as exc:
            failed = True
            errorMsg = 'Activate Manifest - Agent Exception - %s' % exc.getMsg(
            )
            errorCode = exc.getCode()

        except Exception as exc:
            failed = True
            errorMsg = 'Activate Manifest - Unknown error - (%s/%s) - %s - %s' \
                        % (self._service, self._manifest, str(exc), traceback.format_exc(5))
            errorCode = Errors.UNKNOWN_ERROR

        finally:
            if failed:
                if not self._skipCleanupOnFailure():
                    self.__cleanup(symlinkSwitched, errorMsg, errorCode)

                self.__LOG.warning(errorMsg)
                self._updateStatus(httpStatus=500,
                                   error=errorCode,
                                   errorMsg=errorMsg)
            else:
                self._updateProgress(100)
Esempio n. 30
0
    def testDelete(self):
        def deleteTestDir(service):
            LOG.debug('************ service = %s' % service)
            response = self.app.delete(url(controller='service',
                                           service=service,
                                           action='delete'),
                                       expect_errors=True)
            # make sure the responses are correct
            LOG.debug('status = ' + str(response.status_int))
#            assert response.status_int == 500, "HTTP response != 500"

        def makeTestDir(path):
            os.makedirs(path)
            os.makedirs(os.path.join(path, 'manifests'))
            os.makedirs(os.path.join(path, 'installed-packages'))

        def createManifests(mf_path):
            os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
            os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
            latest = os.path.join(mf_path, 'm3.0')
            os.makedirs(os.path.join(latest, 'dummy_dir3.0'))

            utils.symlink(latest, os.path.join(mf_path, 'active'))

            return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')

        def makePackageContent(path, pkgPath, pkgPropPath):
            pkgFile = file(pkgPath, 'w')
            for index in range(10):
                pkgFile.write(('%s%s') % (index, index))
            pkgFile.close()

            pkgFile = file(pkgPropPath, 'w')
            for index in range(10):
                pkgFile.write(('%s%s') % (index, index))
            pkgFile.close()
            uname = pylons.config['agent_user_account']
            TestCleanupController.rchown(path, uname)

        def createTestThread(serviceName):
            appGlobal = config['pylons.app_globals']
            testTh = WaitThread(appGlobal.threadMgr,
                                ServiceController.serviceCat(serviceName))
            testTh.start()
            return testTh

        def startTestProcess():
            cmd = utils.sudoCmd(["sleep", "5"],
                                pylons.config['app_user_account'])
            return Popen(cmd)

        if os.name == 'nt':
            LOG.warning(
                'Services cleanup not supported on windows. Skipping test...')
            return

        path1 = os.path.join(pylons.config['agent_root'], 'service_nodes',
                             'foo')
        path2 = os.path.join(pylons.config['agent_root'], 'service_nodes',
                             'bar')
        path3 = os.path.join(pylons.config['agent_root'], 'service_nodes',
                             'agent')

        deleteTestDir('foo')
        deleteTestDir('bar')
        deleteTestDir('agent')

        # make dirs
        makeTestDir(path1)
        makeTestDir(path2)
        makeTestDir(path3)
        all_mf, active_mf = createManifests(
            ServiceController.manifestPath('agent'))

        uname = pylons.config['agent_user_account']
        TestCleanupController.rchown(ServiceController.serviceRootPath(),
                                     uname)

        pkgDir = PackageMgr.packagePath()
        pkgPath = os.path.join(pkgDir, "foo.cronus")
        pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop")
        makePackageContent(pkgDir, pkgPath, pkgPropPath)

        # create threads
        testThFoo = createTestThread('foo')
        testThBar = createTestThread('bar')
        testThAgent = createTestThread('agent')

        # start process
        process = startTestProcess()

        # start testing
        LOG.debug('************ start cleanup')
        response = self.app.post(url(controller='cleanup', action='post'))
        LOG.debug('Delete response body = ' + response.body)
        body = json.loads(response.body)

        tm = time.time()
        while (tm + 10 > time.time()):
            response = self.app.get(body['status'], expect_errors=True)
            LOG.debug('Status response body = ' + response.body)
            body = json.loads(response.body)
            print body
            if (body['progress'] == 100):
                break
            time.sleep(0.1)

        # make sure the responses are correct
        LOG.debug('status = ' + str(response.status_int))
        assert response.status_int == 200, "HTTP response != 200"

        time.sleep(0.1)
        assert not os.path.exists(
            path1), 'service foo does exist or is not a directory'
        assert not os.path.exists(
            path2), 'service bar does exist or is not a directory'
        assert os.path.exists(
            path3), 'service agent does NOT exist or is not a directory'
        assert not testThFoo.isAlive(), 'thread Foo is still alive'
        assert not testThBar.isAlive(), 'thread Bar is still alive'
        assert not testThAgent.isAlive(), 'thread Agent is still alive'
        assert not os.path.exists(pkgPath), 'package foo exists'
        assert not os.path.exists(pkgPropPath), 'package prop foo exists'
        assert os.path.exists(pkgDir), 'package directory does not exist'
        # ensure agent cleanup is proper
        active_mf_path = ManifestController.manifestPath('agent', active_mf)
        active_link = os.path.join(ServiceController.manifestPath('agent'),
                                   'active')
        all_mf.remove(active_mf)
        actual_active_mf_path = utils.readlink(active_link)

        self.assertTrue(
            os.path.exists(active_mf_path),
            'active agent manifest got deleted but shouldn\t have')
        self.assertTrue(os.path.exists(active_link),
                        'agent active link missing')
        self.assertEqual(
            active_mf_path, actual_active_mf_path,
            'agent active link pointing to some wrong manifest; link broken?')
        for mf in all_mf:
            agnt_mf_path = ManifestController.manifestPath('agent', mf)
            self.assertFalse(
                os.path.exists(agnt_mf_path),
                'non active agent mf %s should have been deleted' % mf)
Esempio n. 31
0
    def doRun(self):
        """ Main body of the thread """
        errorMsg = ""
        errorCode = None
        symlinkSwitched = False
        failed = False
        
        try:
            activePath = manifestutil.manifestPath(self._service)
            oldManifest = None
            
            # make sure that if the active path exists, it's a link
            # if not log that and delete the link
            if (os.path.exists(activePath) and not islink(activePath)):
                self._LOG.error('%s is not a link.  Attempted to delete' % activePath)
                shutil.rmtree(activePath)

            if (os.path.exists(activePath)):
                oldManifest = os.path.basename(readlink(activePath))
            # reset requires a valid active link exist
            else:
                if self.__action == ActivateManifest.ACTION_RESET:
                    raise AgentException(error = Errors.ACTIVEMANIFEST_MANIFEST_MISSING, 
                                         errorMsg = ('No active manifest - cannot %s service' 
                                                     % self.__action))

            if self.__action == ActivateManifest.ACTION_ACTIVATION:
                # install new manifest
                self.__installManifest(self._service, self._manifest)
            
            self._deactivateManifest(self._service, oldManifest, 11, 50)
            
            if self.__action == ActivateManifest.ACTION_ACTIVATION:
                # special logic to handle agent upgrade
                # agent upgrade will not run full activation agent will shutdown before activate exit
                if self._service == 'agent':
                    appGlobal = pylons.config['pylons.app_globals']
                    # persist threads result before shutting down
                    if (hasattr(appGlobal, 'threadMgr') and appGlobal.threadMgr != None):
                        killStatus = {}
                        killStatus['httpStatus'] = 500
                        killStatus['error'] = Errors.THREAD_KILLED_AGENT_RESTART
                        killStatus['errorMsg'] = 'thread killed, agent restart'
                        appGlobal.threadMgr.snapshot(killStatus, True)
            
                # switch active symlink
                symlinkSwitched = self.__switchSymlink(self._service, self._manifest)
                # activate new manifest
                self._activateManifest(self._service, self._manifest, 51, 80)

            elif self.__action == ActivateManifest.ACTION_DEACTIVATION:
                # remove active link on deactivate
                activePath = self.__getSymlinkPath(self._service)
                self.__removeSymlink(activePath)

            else:
                # activate new manifest
                self._activateManifest(self._service, self._manifest, 51, 80)

        except SystemExit as exc:
            failed = True
            if (len(exc.args) == 2):
                # ok we got {err code, err msg}
                errorCode = exc.args[0]
                errorMsg = exc.args[1]
            raise exc
        
        except AgentException as exc:
            failed = True
            errorMsg = '%s manifest - Agent Exception - %s' % (self.__action, exc.getMsg())
            errorCode = exc.getCode()
        
        except Exception as exc:
            failed = True
            errorMsg = '%s manifest - Unknown error - (%s/%s) - %s - %s' \
                        % (self.__action, self._service, self._manifest, 
                           str(exc), traceback.format_exc(5))
            errorCode = Errors.UNKNOWN_ERROR
        
        finally:
            if failed: 
                if not self._skipCleanupOnFailure():
                    self.__cleanup(symlinkSwitched, errorMsg, errorCode)

                self._LOG.warning(errorMsg)
                self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
            else:
                self._updateProgress(100)
Esempio n. 32
0
    def testDelete(self):
        def deleteTestDir(service):
            LOG.debug('************ service = %s' % service)
            response = self.app.delete(url(controller='service', service=service, action='delete'), expect_errors = True)
            # make sure the responses are correct
            LOG.debug('status = ' + str(response.status_int))
#            assert response.status_int == 500, "HTTP response != 500"

        def makeTestDir(path):
            os.makedirs(path)
            os.makedirs(os.path.join(path, 'manifests'))
            os.makedirs(os.path.join(path, 'installed-packages'))

        def createManifests(mf_path):
            os.makedirs(os.path.join(mf_path, 'm1.0', 'dummy_dir1.0'))
            os.makedirs(os.path.join(mf_path, 'm2.0', 'dummy_dir2.0'))
            latest = os.path.join(mf_path, 'm3.0')
            os.makedirs(os.path.join(latest, 'dummy_dir3.0'))

            utils.symlink(latest, os.path.join(mf_path, 'active'))

            return (['m1.0', 'm2.0', 'm3.0'], 'm3.0')

        def makePackageContent(path, pkgPath, pkgPropPath):
            pkgFile = file(pkgPath, 'w')
            for index in range(10):
                pkgFile.write(('%s%s') % (index, index))
            pkgFile.close()

            pkgFile = file(pkgPropPath, 'w')
            for index in range(10):
                pkgFile.write(('%s%s') % (index, index))
            pkgFile.close()
            uname = pylons.config['agent_user_account']
            TestCleanupController.rchown(path, uname)

        def createTestThread(serviceName):
            appGlobal = config['pylons.app_globals']
            testTh = WaitThread(appGlobal.threadMgr, ServiceController.serviceCat(serviceName))
            testTh.start()
            return testTh

        def startTestProcess():
            cmd = utils.sudoCmd(["sleep", "5"], pylons.config['app_user_account'])
            return Popen(cmd)

        path1 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'foo')
        path2 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'bar')
        path3 = os.path.join(pylons.config['agent_root'], 'service_nodes', 'agent')

        deleteTestDir('foo')
        deleteTestDir('bar')
        deleteTestDir('agent')

        # make dirs
        makeTestDir(path1)
        makeTestDir(path2)
        makeTestDir(path3)
        all_mf, active_mf = createManifests(ServiceController.manifestPath('agent'))

        uname = pylons.config['agent_user_account']
        TestCleanupController.rchown(ServiceController.serviceRootPath(), uname)

        pkgDir = PackageMgr.packagePath()
        pkgPath = os.path.join(pkgDir, "foo.cronus")
        pkgPropPath = os.path.join(pkgDir, "foo.cronus.prop")
        makePackageContent(pkgDir, pkgPath, pkgPropPath)

        # create threads
        testThFoo = createTestThread('foo')
        testThBar = createTestThread('bar')
        testThAgent = createTestThread('agent')

        # start process
        process = startTestProcess()

        # start testing
        LOG.debug('************ start cleanup')
        response = self.app.post(url(controller='cleanup', action='post'))
        LOG.debug ('Delete response body = ' + response.body)
        body = json.loads(response.body)

        tm = time.time()
        while (tm + 10 > time.time()):
            response = self.app.get(body['status'], expect_errors = True)
            LOG.debug ('Status response body = ' + response.body)
            body = json.loads(response.body)
            print body
            if (body['progress'] == 100):
                break
            time.sleep(0.1)

        # make sure the responses are correct
        LOG.debug('status = ' + str(response.status_int))
        assert response.status_int == 200, "HTTP response != 200"

        time.sleep(0.1)
        assert not os.path.exists(path1), 'service foo does exist or is not a directory'
        assert not os.path.exists(path2), 'service bar does exist or is not a directory'
        assert os.path.exists(path3), 'service agent does NOT exist or is not a directory'
        assert not testThFoo.isAlive(), 'thread Foo is still alive'
        assert not testThBar.isAlive(), 'thread Bar is still alive'
        assert not testThAgent.isAlive(), 'thread Agent is still alive'
        assert not os.path.exists(pkgPath), 'package foo exists'
        assert not os.path.exists(pkgPropPath), 'package prop foo exists'
        assert os.path.exists(pkgDir), 'package directory does not exist'
        # ensure agent cleanup is proper
        active_mf_path = manifestutil.manifestPath('agent', active_mf)
        active_link = os.path.join(ServiceController.manifestPath('agent'), 'active')
        all_mf.remove(active_mf)
        actual_active_mf_path = utils.readlink(active_link)

        self.assertTrue(os.path.exists(active_mf_path), 'active agent manifest got deleted but shouldn\t have')
        self.assertTrue(os.path.exists(active_link), 'agent active link missing')
        self.assertEqual(active_mf_path, actual_active_mf_path, 'agent active link pointing to some wrong manifest; link broken?')
        for mf in all_mf:
            agnt_mf_path = manifestutil.manifestPath('agent', mf)
            self.assertFalse(os.path.exists(agnt_mf_path), 'non active agent mf %s should have been deleted' % mf)