Example #1
0
    def post(self, service, manifest):
        """ Create a new service object """
        from agent.lib.agent_thread.manifest_create import ManifestCreate

        try:

            LOG.info('Post for service (%s) and manifest (%s) with body: %s',
                      service, manifest, request.body)

            # check to see if the manifest already exists
            path = ManifestController.manifestPath(service, manifest)
            if (os.path.isdir(path)):
                return doneResult(request, response, httpStatus = 201, controller = self)

            # parse the body
            if (request.body == ""):
                return errorResult(request, response, Errors.MANIFEST_PACKAGE_PARSING_ERROR,
                                   'No body found in post command',
                                   controller = self)

            body = json.loads(request.body)

            packages = body['package']
            forcedPackages = body['forcePackageName'] if 'forcePackageName' in body else None
            skipProp = asbool(body['skipProp']) if 'skipProp' in body else configutil.getConfigAsBool('download_skip_prop') 

            LOG.debug('pkgs = %s, %s', packages, forcedPackages)

            # parse the package list
            for idx, package in enumerate(packages):
                # to support reuse of an package from an existing manifest (active if possible)
                # without sending the complete package location in request body
                if package.startswith('/'):
                    packageRef = package
                    tokens = package.split('/')
                    pkgnamePrefix = tokens[-1].rstrip()
                    fullPkgLoc = manifestutil.getPackageByName(service, manifest = None, pkgnamePrefix = pkgnamePrefix)
                    if fullPkgLoc is None:
                        return errorResult(request, response, Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
                                           'manifest (%s/%s) package (%s) does not exist' %
                                           (service, manifest, packages), controller = self)
                    else:
                        LOG.info('expanding package reuse ref %s with full package location %s' % (packageRef, fullPkgLoc))
                        packages[idx] = fullPkgLoc

            appGlobal = config['pylons.app_globals']
            # start a thread to create the package
            manThread = ManifestCreate(appGlobal.threadMgr, service, manifest, packages, forcePackages = forcedPackages, skipProp = skipProp)
            self.injectJobCtx(manThread)
            manThread.start()
            manThread.threadMgrEvent.wait()

            return statusResult(request, response, manThread, controller = self)
        except AgentException as excep:
            return errorResult(request, response, error = excep.getCode(), errorMsg = excep.getMsg(), controller = self)
        except Exception as excep:
            return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
                               errorMsg = 'Unknown error for activateManifest(%s/%s) - %s - %s' %
                               (service, manifest, str(excep), traceback.format_exc(2)),
                               controller = self)
Example #2
0
    def test_post_manifest_inprogress_ok(self):
        service = 'foo'
        manifest = 'blahblah'
        try:
            path = ServiceController.servicePath(service)
            if os.path.exists(path):
                if os.name == 'nt':
                    cmd = 'rm -r %s' % path
                    LOG.debug("running command %s" % cmd)
                    os.system(cmd)
                else:
                    shutil.rmtree(path)
            path = ServiceController.manifestPath(service)
            os.makedirs(path)
            path = ServiceController.installedPkgPath(service)
            os.makedirs(path)
            inProgressPath = ManifestCreate.inProgress(ManifestController.manifestPath(service, manifest))
            os.makedirs(inProgressPath)
            path = ServiceController.downloadedPkgPath(service)
            os.makedirs(path)
        except Exception as excep:
            LOG.warning('got an OS Exception - %s' % str(excep))

        createManifest(self, ["http://github.com/yubin154/cronusagent/blob/master/agent/agent/tests/unit/packages/pkgA-1.2.0.unix.cronus"], manifest = 'blahblah', createDirs = False)
        self.assertTrue(islink(os.path.join(ManifestController.manifestPath('foo', 'blahblah'), 'pkgA')))
Example #3
0
    def __createManifest(self):
        """ create a manifest
        """
        service = 'agent'
        LOG.info("Create Manifest %s - %s - %s" % (service, self.__manifest, str(self.__packages)))
        path = manifestutil.manifestPath(service, self.__manifest)
        # check to see if the manifest already exists
        if (os.path.isdir(path)):
            LOG.info('Manifest %s already exist, skip creating' % self.__manifest)
            return

        from agent.lib.agent_thread.manifest_create import ManifestCreate
        manThread = ManifestCreate(self._threadMgr, service, self.__manifest, self.__packages, skipProp = self.__skipProp)
        contextutils.copycontexts(self, manThread, contextutils.CTX_NAMES)
        manThread.run()
        status = manThread.getStatus()
        if (status.has_key('error') and status['error']):
            raise AgentException(status['error'], status['errorMsg'])
Example #4
0
    def get(self, service):
        """ Get a new service object """
        try:
            from agent.lib.agent_thread.manifest_create import ManifestCreate

            # make sure the service path exists
            path = ServiceController.servicePath(service)
            if (not os.path.exists(path)):
                return errorResult(request,
                                   response,
                                   error=Errors.SERVICE_NOT_FOUND,
                                   errorMsg='Unable to find service (%s)' %
                                   service,
                                   controller=self)

            path = ServiceController.manifestPath(service)

            activeManifest = None
            manifestList = []
            for manifest in os.listdir(path):
                if (ManifestCreate.isInProgress(manifest)):
                    continue

                manifestPath = os.path.join(path, manifest)
                if (manifest == 'active'):
                    activeLink = readlink(manifestPath)

                    if (activeLink == None):
                        manifestList.append(manifest)
                    else:
                        activeManifest = os.path.basename(activeLink)
                else:
                    manifestList.append(manifest)

            result = {}
            manifestList.sort()
            result['manifest'] = manifestList
            result['activemanifest'] = activeManifest

            return doneResult(request,
                              response,
                              result=result,
                              controller=self)

        except Exception as excep:
            return errorResult(
                request,
                response,
                error=Errors.UNKNOWN_ERROR,
                errorMsg='Unknown error when getting service (%s) %s - %s' %
                (service, str(excep), traceback.format_exc(2)),
                controller=self)
Example #5
0
    def get(self, service):
        """ Get a new service object """
        try:
            from agent.lib.agent_thread.manifest_create import ManifestCreate

            # make sure the service path exists
            path = ServiceController.servicePath(service)
            if not os.path.exists(path):
                return errorResult(
                    request,
                    response,
                    error=Errors.SERVICE_NOT_FOUND,
                    errorMsg="Unable to find service (%s)" % service,
                    controller=self,
                )

            path = ServiceController.manifestPath(service)

            activeManifest = None
            manifestList = []
            for manifest in os.listdir(path):
                if ManifestCreate.isInProgress(manifest):
                    continue

                manifestPath = os.path.join(path, manifest)
                if manifest == "active":
                    activeLink = readlink(manifestPath)

                    if activeLink == None:
                        manifestList.append(manifest)
                    else:
                        activeManifest = os.path.basename(activeLink)
                else:
                    manifestList.append(manifest)

            result = {}
            manifestList.sort()
            result["manifest"] = manifestList
            result["activemanifest"] = activeManifest

            return doneResult(request, response, result=result, controller=self)

        except Exception as excep:
            return errorResult(
                request,
                response,
                error=Errors.UNKNOWN_ERROR,
                errorMsg="Unknown error when getting service (%s) %s - %s"
                % (service, str(excep), traceback.format_exc(2)),
                controller=self,
            )
Example #6
0
    def __createManifest(self):
        """ create a manifest
        """
        service = 'agent'
        LOG.info("Create Manifest %s - %s - %s" %
                 (service, self.__manifest, str(self.__packages)))
        path = manifestutil.manifestPath(service, self.__manifest)
        # check to see if the manifest already exists
        if (os.path.isdir(path)):
            LOG.info('Manifest %s already exist, skip creating' %
                     self.__manifest)
            return

        from agent.lib.agent_thread.manifest_create import ManifestCreate
        manThread = ManifestCreate(self._threadMgr,
                                   service,
                                   self.__manifest,
                                   self.__packages,
                                   skipProp=self.__skipProp)
        contextutils.copycontexts(self, manThread, contextutils.CTX_NAMES)
        manThread.run()
        status = manThread.getStatus()
        if (status.has_key('error') and status['error']):
            raise AgentException(status['error'], status['errorMsg'])
Example #7
0
    def getManifests(service):
        """ return the list of manifests under a specific service """
        from agent.lib.agent_thread.manifest_create import ManifestCreate

        manifests = []

        dirContent = os.listdir(ServiceController.manifestPath(service))

        for item in dirContent:
            path = os.path.join(ServiceController.manifestPath(service), item)
            if (os.path.isdir(path) and not ManifestCreate.isInProgress(path)):
                if (not islink(path)):
                    manifests.append(item)

        return sorted(manifests)
Example #8
0
    def getManifests(service):
        """ return the list of manifests under a specific service """
        from agent.lib.agent_thread.manifest_create import ManifestCreate

        manifests = []

        dirContent = os.listdir(ServiceController.manifestPath(service))

        for item in dirContent:
            path = os.path.join(ServiceController.manifestPath(service), item)
            if (os.path.isdir(path) and not ManifestCreate.isInProgress(path)):
                if (not islink(path)):
                    manifests.append(item)

        return sorted(manifests)
Example #9
0
    def cleanupServices():
        """
        for all services clean up partially completed manifests
        This should only be called during startup.
        """
        from agent.lib.agent_thread.manifest_create import ManifestCreate

        try:
            services = ServiceController.getServices()
            for service in services:
                manPath = ServiceController.manifestPath(service)
                manifestPaths = [os.path.join(manPath, path) for path in os.listdir(manPath)]

                [shutil.rmtree(manifestPath) for manifestPath in manifestPaths
                 if ManifestCreate.isInProgress(os.path.basename(manifestPath))]

        except OSError:
            pass
Example #10
0
    def cleanupServices():
        """
        for all services clean up partially completed manifests
        This should only be called during startup.
        """
        from agent.lib.agent_thread.manifest_create import ManifestCreate

        try:
            services = ServiceController.getServices()
            for service in services:
                manPath = ServiceController.manifestPath(service)
                manifestPaths = [
                    os.path.join(manPath, path) for path in os.listdir(manPath)
                ]

                [
                    shutil.rmtree(manifestPath)
                    for manifestPath in manifestPaths
                    if ManifestCreate.isInProgress(
                        os.path.basename(manifestPath))
                ]

        except OSError:
            pass
Example #11
0
    def doRun(self):
        """ Main body of the thread """
        self._updateProgress(1)

        errorMsg = ""
        errorCode = None
        failed = False
        
        try:
            # create manifest if not already exist
            mpath = manifestutil.manifestPath(self._service, self._manifest)
            if (not os.path.exists(mpath) or not os.path.isdir(mpath)):
                self._LOG.debug('pkgs = %s', self.__packages)

                # parse the package list
                for idx, package in enumerate(self.__packages):
                    if package.startswith('/'):
                        packageRef = package
                        tokens = package.split('/')
                        pkgnamePrefix = tokens[-1].rstrip()
                        fullPkgLoc = manifestutil.getPackageFullLocByName(self._service, manifest = None, pkgnamePrefix = pkgnamePrefix)
                        if fullPkgLoc is None:
                            raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST, 
                                                 'manifest (%s/%s) package (%s) does not exist' % 
                                                 (self._service, self._manifest, self.__packages))
                        else:
                            self._LOG.info('expanding package reuse ref %s with full package location %s' % (packageRef, fullPkgLoc))
                            self.__packages[idx] = fullPkgLoc

                # start a thread to create the package
                manThread = ManifestCreate(self._threadMgr, self._service, self._manifest, self.__packages, parentId = self.getUuid())
                contextutils.copyJobContexts(self, manThread)
                manThread.run()
                
                self._addChildExeThreadId(manThread.getChildExeThreadIds())

                status = manThread.getStatus()
                if (status['error'] != None):
                    raise AgentException(status['error'], status['errorMsg'])
                
            
            self._updateProgress(60)
            if (not os.path.exists(mpath)):
                raise AgentException(Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
                                'Manifest(%s, %s) path missing' % (self._service, self._manifest))

            # activate manifest if not already activated
            activateThread = ActivateManifest(self._threadMgr, self._service, self._manifest, parentId = self.getUuid())
            contextutils.copyJobContexts(self, activateThread)
            activateThread.run()
            
            self._addChildExeThreadId(activateThread.getChildExeThreadIds())

            status = activateThread.getStatus()
            if (status['error'] != None):
                raise AgentException(status['error'], status['errorMsg'])

            if manifestutil.getActiveManifest(self._service) != self._manifest:
                raise AgentException(Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
                                    'Manifest(%s, %s) path missing' % (self._service, self._manifest))

        except SystemExit as exc:
            failed = True
            if (len(exc.args) == 2):
                # ok we got {err code, err msg}
                errorCode = exc.args[0]
                errorMsg = exc.args[1]
        
        except AgentException as exc:
            failed = True
            errorMsg = 'Deploy Service - Agent Exception - %s' % exc.getMsg()
            errorCode = exc.getCode()
        
        except Exception as exc:
            failed = True
            errorMsg = 'Deploy Service - Unknown error - (%s/%s) - %s - %s' \
                        % (self._service, self._manifest, str(exc), traceback.format_exc(5))
            errorCode = Errors.UNKNOWN_ERROR
        
        finally:
            if failed: 
                self._LOG.warning(errorMsg)
                self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
            else:
                self._updateProgress(100)
Example #12
0
    def post(self, service, manifest):
        """ Create a new service object """
        from agent.lib.agent_thread.manifest_create import ManifestCreate

        try:

            LOG.info('Post for service (%s) and manifest (%s) with body: %s',
                     service, manifest, request.body)

            # check to see if the manifest already exists
            path = ManifestController.manifestPath(service, manifest)
            if (os.path.isdir(path)):
                return doneResult(request,
                                  response,
                                  httpStatus=201,
                                  controller=self)

            # parse the body
            if (request.body == ""):
                return errorResult(request,
                                   response,
                                   Errors.MANIFEST_PACKAGE_PARSING_ERROR,
                                   'No body found in post command',
                                   controller=self)

            body = json.loads(request.body)

            packages = body['package']
            forcedPackages = body[
                'forcePackageName'] if 'forcePackageName' in body else None
            skipProp = asbool(
                body['skipProp']
            ) if 'skipProp' in body else configutil.getConfigAsBool(
                'download_skip_prop')

            LOG.debug('pkgs = %s, %s', packages, forcedPackages)

            # parse the package list
            for idx, package in enumerate(packages):
                # to support reuse of an package from an existing manifest (active if possible)
                # without sending the complete package location in request body
                if package.startswith('/'):
                    packageRef = package
                    tokens = package.split('/')
                    pkgnamePrefix = tokens[-1].rstrip()
                    fullPkgLoc = manifestutil.getPackageByName(
                        service, manifest=None, pkgnamePrefix=pkgnamePrefix)
                    if fullPkgLoc is None:
                        return errorResult(
                            request,
                            response,
                            Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST,
                            'manifest (%s/%s) package (%s) does not exist' %
                            (service, manifest, packages),
                            controller=self)
                    else:
                        LOG.info(
                            'expanding package reuse ref %s with full package location %s'
                            % (packageRef, fullPkgLoc))
                        packages[idx] = fullPkgLoc

            appGlobal = config['pylons.app_globals']
            # start a thread to create the package
            manThread = ManifestCreate(appGlobal.threadMgr,
                                       service,
                                       manifest,
                                       packages,
                                       forcePackages=forcedPackages,
                                       skipProp=skipProp)
            self.injectJobCtx(manThread)
            manThread.start()
            manThread.threadMgrEvent.wait()

            return statusResult(request, response, manThread, controller=self)
        except AgentException as excep:
            return errorResult(request,
                               response,
                               error=excep.getCode(),
                               errorMsg=excep.getMsg(),
                               controller=self)
        except Exception as excep:
            return errorResult(
                request,
                response,
                error=Errors.UNKNOWN_ERROR,
                errorMsg='Unknown error for activateManifest(%s/%s) - %s - %s'
                % (service, manifest, str(excep), traceback.format_exc(2)),
                controller=self)
Example #13
0
    def doRun(self):
        """ Main body of the thread """
        spath = manifestutil.servicePath(self._service)

        self._updateProgress(1)

        errorMsg = ""
        errorCode = None
        failed = False
        ctxNames = ['guid', 'service']
        
        try:
            # create service if not already exist
            if not os.path.exists(spath):
                os.makedirs(spath)
                os.makedirs(os.path.join(spath, 'manifests'))
                os.makedirs(os.path.join(spath, 'installed-packages'))
                os.makedirs(os.path.join(spath, 'modules'))
                os.makedirs(os.path.join(spath, 'downloaded-packages'))
                os.makedirs(os.path.join(spath, '.appdata'))
                os.makedirs(os.path.join(spath, '.data'))
                import pwd
                uname = pylons.config['app_user_account']
                uid = pwd.getpwnam(uname).pw_uid
                gid = pwd.getpwnam(uname).pw_gid
                utils.rchown(os.path.join(spath, '.appdata'), uid, gid)

            # verify that the path exists
            if (not os.path.isdir(spath)):
                raise AgentException(Errors.UNKNOWN_ERROR, "Service(%s) was not created" % self._service)

            self._updateProgress(20)

            # create manifest if not already exist
            mpath = manifestutil.manifestPath(self._service, self._manifest)
            if (not os.path.exists(mpath) or not os.path.isdir(mpath)):
                self.__LOG.debug('pkgs = %s', self.__packages)

                # parse the package list
                for idx, package in enumerate(self.__packages):
                    if package.startswith('/'):
                        packageRef = package
                        tokens = package.split('/')
                        pkgnamePrefix = tokens[-1].rstrip()
                        fullPkgLoc = manifestutil.getPackageByName(self._service, manifest = None, pkgnamePrefix = pkgnamePrefix)
                        if fullPkgLoc is None:
                            raise AgentException(Errors.MANIFEST_PACKAGE_DOES_NOT_EXIST, 
                                                 'manifest (%s/%s) package (%s) does not exist' % 
                                                 (self._service, self._manifest, self.__packages))
                        else:
                            self.__LOG.info('expanding package reuse ref %s with full package location %s' % (packageRef, fullPkgLoc))
                            self.__packages[idx] = fullPkgLoc

                # start a thread to create the package
                manThread = ManifestCreate(threadmgr.NULL_THREADMGR, self._service, self._manifest, self.__packages, skipProp = self.__skipProp)
                contextutils.copycontexts(self, manThread, ctxNames)
                manThread.run()
                status = manThread.getStatus()
                if (status['error'] != None):
                    raise AgentException(status['error'], status['errorMsg'])
                
            
            self._updateProgress(60)
            if (not os.path.exists(mpath)):
                raise AgentException(Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
                                'Manifest(%s, %s) path missing' % (self._service, self._manifest))

            if not self.__skipActivation:                    
                activateThread = ActivateManifest(threadmgr.NULL_THREADMGR, self._service, self._manifest)
                contextutils.copycontexts(self, activateThread, ctxNames)
                activateThread.run()
            
                status = activateThread.getStatus()
                if (status['error'] != None):
                    raise AgentException(status['error'], status['errorMsg'])

                # activte manifest if not already activated
                if manifestutil.getActiveManifest(self._service) != self._manifest:
                    raise AgentException(Errors.ACTIVEMANIFEST_MANIFEST_MISSING,
                                         'Manifest(%s, %s) path missing' % (self._service, self._manifest))

        except SystemExit as exc:
            failed = True
            if (len(exc.args) == 2):
                # ok we got {err code, err msg}
                errorCode = exc.args[0]
                errorMsg = exc.args[1]
        
        except AgentException as exc:
            failed = True
            errorMsg = 'Deploy Service - Agent Exception - %s' % exc.getMsg()
            errorCode = exc.getCode()
        
        except Exception as exc:
            failed = True
            errorMsg = 'Deploy Service - Unknown error - (%s/%s) - %s - %s' \
                        % (self._service, self._manifest, str(exc), traceback.format_exc(5))
            errorCode = Errors.UNKNOWN_ERROR
        
        finally:
            if failed: 
                self.__LOG.warning(errorMsg)
                self._updateStatus(httpStatus = 500, error = errorCode, errorMsg = errorMsg)
            else:
                self._updateProgress(100)