示例#1
0
    def _untarPackages(self, packages, service, untarRootPath, nicelevel=None, pgksNeedSuffix=None, pathSuffix=''):
        """
        untar all the packages

        @params packages - list of dictionary of parsed packages uri
        @throws AgentException
        """
        self._updateProgress(80)
        count = 0
        for pkgDict in packages.itervalues():
            count += 1

            # check if package path exists already
            # if the package is already untarred, move on
            # else create the package path
            pkgName = pkgDict['packageName']
            untarPath = os.path.join(untarRootPath,
                                    pkgName,
                                    '%s%s' % (pkgDict['packageVersion'], pathSuffix if ((pgksNeedSuffix is not None) and pkgName in pgksNeedSuffix) else ''))

            if os.path.exists(untarPath):
                # perhaps another thread has finished extraction, continue with next package
                continue
            
            os.makedirs(untarPath)

            try:
                self.untar(pkgDict['packagePath'], untarPath, nicelevel)

            except AgentException:
                rmrf(untarPath)

            # Note: i. atleast self should have 'rx' so that we can proceed setting 'rx' for group and others
            # ii. since cronus/cronusapp belong to diff group, lets give access to self, group and others.
            # if both belong to same group in future, then just self, group should be enough
            if (not os.name == 'nt'):
                # ensure all parent dir of scripts dir have 'rx' so that we can really navigate to scripts dir and execute
                uname = getuserofpath(untarPath)
                chmod(untarPath, '+rx', sudoUser = uname)
                cronusPath = os.path.join(untarPath, 'cronus')
                if os.path.exists(cronusPath):
                    uname = getuserofpath(cronusPath)
                    chmod(cronusPath, '+rx', sudoUser = uname)
                    # now give all scripts 'rx' permission
                    scriptsPath = os.path.join(cronusPath, 'scripts')
                    if os.path.exists(scriptsPath):
                        uname = getuserofpath(scriptsPath)
                        rchmod(scriptsPath, '+rx', sudoUser = uname)

                #Running as cronus user when higher privilege service (i.e. not chown all the package into the application user)
                if (not isHigherPrivilegeService(service)):
                    import pwd
                    uname = pylons.config['app_user_account']
                    uid = pwd.getpwnam(uname).pw_uid
                    gid = pwd.getpwnam(uname).pw_gid
                    rchown(untarPath, uid, gid)

            self._updateProgress(calcProgress(80, 99, float(count) / len(packages)))
示例#2
0
    def post(self, service):
        """ Create a new service object """
        try:

            LOG.info('Got a post request for service ' + service)
            path = ServiceController.servicePath(service)

            # skip if the path already exists
            if os.path.exists(path) and os.path.isdir(path):
                return doneResult(request,
                                  response,
                                  httpStatus=201,
                                  result=service,
                                  controller=self)

            os.makedirs(path)
            os.makedirs(os.path.join(path, 'manifests'))
            os.makedirs(os.path.join(path, 'installed-packages'))
            os.makedirs(os.path.join(path, 'modules'))
            os.makedirs(os.path.join(path, 'downloaded-packages'))
            os.makedirs(os.path.join(path, '.appdata'))
            os.makedirs(os.path.join(path, '.data'))
            utils.rchmod(os.path.join(path, '.appdata'), '+rwx')

            # verify that the path exists
            if (not os.path.isdir(path)):
                return errorResult(request,
                                   response,
                                   Errors.UNKNOWN_ERROR,
                                   "Service(%s) was not created" % service,
                                   controller=self)

            return doneResult(request,
                              response,
                              result=service,
                              controller=self)

        except OSError:
            return errorResult(request,
                               response,
                               Errors.SERVICE_EXISTS,
                               "Service(%s) already exists(or %s)" %
                               (service, traceback.format_exc(2)),
                               controller=self)
        except Exception as excep:
            return errorResult(
                request,
                response,
                error=Errors.UNKNOWN_ERROR,
                errorMsg='Unknown error when posting services %s - %s' %
                (str(excep), traceback.format_exc(2)),
                controller=self)
示例#3
0
    def post(self, service):
        """ Create a new service object """
        try:

            LOG.info("Got a post request for service " + service)
            path = ServiceController.servicePath(service)

            # skip if the path already exists
            if os.path.exists(path) and os.path.isdir(path):
                return doneResult(request, response, httpStatus=201, result=service, controller=self)

            os.makedirs(path)
            os.makedirs(os.path.join(path, "manifests"))
            os.makedirs(os.path.join(path, "installed-packages"))
            os.makedirs(os.path.join(path, "modules"))
            os.makedirs(os.path.join(path, "downloaded-packages"))
            os.makedirs(os.path.join(path, ".appdata"))
            os.makedirs(os.path.join(path, ".data"))
            utils.rchmod(os.path.join(path, ".appdata"), "+rwx")

            # verify that the path exists
            if not os.path.isdir(path):
                return errorResult(
                    request, response, Errors.UNKNOWN_ERROR, "Service(%s) was not created" % service, controller=self
                )

            return doneResult(request, response, result=service, controller=self)

        except OSError:
            return errorResult(
                request,
                response,
                Errors.SERVICE_EXISTS,
                "Service(%s) already exists(or %s)" % (service, traceback.format_exc(2)),
                controller=self,
            )
        except Exception as excep:
            return errorResult(
                request,
                response,
                error=Errors.UNKNOWN_ERROR,
                errorMsg="Unknown error when posting services %s - %s" % (str(excep), traceback.format_exc(2)),
                controller=self,
            )
示例#4
0
    def __startDownload(self):
        """ actual download logic """
        try:
            LOG.info("Starting package download for package %s" %
                     self.__uriDict['package'])

            # check to see if there's an in progress file,
            # since PackageMgr guarantees that duplicate threads will not be spawned
            # for same pkg, assume an existing thread was killed.
            # attempt to clean up package n move
            if (os.path.exists(self.__uriDict['inProgressPackagePath'])):
                LOG.debug(
                    'In progress file (%s) already exists. Will validate and reattempt download if necessary'
                    % self.__uriDict['inProgressPackagePath'])

            if os.path.exists(self.__uriDict['packagePath']):
                if (os.path.exists(self.__uriDict['propPath']) and
                        PackageUtil.validateProp(self.__uriDict['propPath'])
                        and PackageUtil.validatePackage(
                            self.__uriDict['packagePath'],
                            self.__uriDict['propPath'])):
                    msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict[
                        'packagePath']
                    LOG.info(msg)
                    os.utime(self.__uriDict['packagePath'], None)
                    os.utime(self.__uriDict['propPath'], None)
                    self._updateStatus(progress=100)
                    # NOTE: this is a normal exit not an error!
                    return
                LOG.warning(
                    'The package already exists. However package prop (%s) failed validation. Downloading package.'
                    % self.__uriDict['propPath'])

            # Delete all traces of package before beginning download
            LOG.info('Cleaning up all packages for %s ' %
                     self.__uriDict['packagePath'])
            PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'],
                                       self.__uriDict['packagePath'],
                                       self.__uriDict['propPath'])

            AgentThread._updateProgress(self, 0)

            if not self.__skipProp:
                # First, download .prop file
                LOG.info(
                    'Starting download of prop file %s - %s' %
                    (self.__uriDict['propUri'], self.__uriDict['propPath']))
                self.__download_prop_file()
                try:
                    self.__prop = loadPropFile(self.__uriDict['propPath'])
                except FileNotFoundError:
                    raise AgentException(
                        Errors.DC_MISSING_PROP_FILE,
                        'Prop file (%s) unable to read or did not parse' %
                        (self.__uriDict['propPath']))
            AgentThread._updateProgress(self, 2)

            self.__setProgressTimeouts()

            if self.__uriDict['scheme'] == 'http':
                # try download 3 times, with random sleep
                for _ in range(3):
                    try:
                        sotimeout = float(
                            pylons.config['download_thread_sotimeout'])
                        proxies = json.loads(
                            pylons.config['urlgrabber_proxies'])
                        urlgrabber.urlgrab(
                            self.__uriDict['uri'],
                            self.__uriDict['inProgressPackagePath'],
                            checkfunc=None if self.__skipProp else
                            (PackageUtil.validateDownload, (), {}),
                            progress_obj=DownloadProgress(self),
                            throttle=float(pylons.config['package_throttle']),
                            bandwidth=int(pylons.config['package_bandwidth']),
                            keepalive=0,
                            timeout=sotimeout,
                            proxies=proxies)
                        break
                    except Exception as exc:
                        msg = 'Download error %s - %s' % (
                            str(exc), traceback.format_exc(3))
                        LOG.warning(msg)
                        randsleep = randint(30, 60)
                        time.sleep(randsleep)

            else:
                # oops! only http and bittorrent supported now
                raise AgentException(
                    Errors.DC_UNSUPPORTED_PROTOCOL,
                    'Only http protocols is supported at the moment')

            self._checkStop()

            if not self.__skipProp:
                if (not PackageUtil.validatePackage(
                        self.__uriDict['inProgressPackagePath'],
                        self.__uriDict['propPath'])):
                    raise AgentException(
                        Errors.DC_FAILED_VALIDATE, 'Package ' +
                        self.__uriDict['packagePath'] + ' failed validation')
                os.utime(self.__uriDict['propPath'], None)
                utils.rchmod(self.__uriDict['propPath'], "777", 'no')

            LOG.info(
                'Download complete, will now rename and do validation on this file %s'
                % self.__uriDict['packagePath'])
            os.rename(self.__uriDict['inProgressPackagePath'],
                      self.__uriDict['packagePath'])
            os.utime(self.__uriDict['packagePath'], None)
            utils.rchmod(self.__uriDict['packagePath'], "777", 'no')
            LOG.info(
                "Download complete, Validation completed, updating progress to 100"
            )
            self._updateStatus(progress=100)

        except AgentException, exc:
            self._updateStatus(httpStatus=500,
                               progress=0,
                               error=exc.getCode(),
                               errorMsg=exc.getMsg())
            msg = 'Download error %s - %s' % (str(exc),
                                              traceback.format_exc(3))
            LOG.error(msg)
            raise exc
示例#5
0
    def __startDownload(self):
        """ actual download logic """
        try:
            LOG.info("Starting package download for package %s" % self.__uriDict['package'])

            # check to see if there's an in progress file,
            # since PackageMgr guarantees that duplicate threads will not be spawned
            # for same pkg, assume an existing thread was killed.
            # attempt to clean up package n move
            if (os.path.exists(self.__uriDict['inProgressPackagePath'])):
                LOG.debug('In progress file (%s) already exists. Cleanup and reattempt download' 
                          % self.__uriDict['inProgressPackagePath'])


            if os.path.exists(self.__uriDict['packagePath']):
                if self.__skipProp or ((os.path.exists(self.__uriDict['propPath']) and
                                        PackageUtil.validateProp(self.__uriDict['propPath']) and
                                        PackageUtil.validatePackage(self.__uriDict['packagePath'], 
                                                                    self.__uriDict['propPath']))):
                    msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict['packagePath']
                    LOG.info(msg)
                    os.utime(self.__uriDict['packagePath'], None)
                    if os.path.exists(self.__uriDict['propPath']):
                        os.utime(self.__uriDict['propPath'], None)
                    self._updateStatus(progress = 100)
                    # NOTE: this is a normal exit not an error!
                    return
                else:
                    LOG.warning('The package already exists. However package prop (%s) failed validation.' 
                                % self.__uriDict['propPath'])

            # Delete all traces of package before beginning download
            LOG.debug('Cleaning up all packages for %s ' % self.__uriDict['packagePath'])
            PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'],
                                   self.__uriDict['packagePath'],
                                   self.__uriDict['propPath'])

            AgentThread._updateProgress(self, 0)
            
            if self.__skipProp:
                LOG.info('Skip download of prop file')
            else:
                # First, download .prop file
                LOG.info('Starting download of prop file %s - %s' % (self.__uriDict['propUri'], self.__uriDict['propPath']))
                self.__download_prop_file()
                try:
                    self.__prop = loadPropFile(self.__uriDict['propPath'])
                except FileNotFoundError:
                    raise AgentException(Errors.DC_MISSING_PROP_FILE,
                                         'Prop file (%s) unable to read or did not parse' % (self.__uriDict['propPath']))
                    
            AgentThread._updateProgress(self, 2)
            self.__setProgressTimeouts()

            if self.__uriDict['scheme'] == 'http':
                # try download 3 times, with random sleep
                attempt = configutil.getConfigAsInt('download_thread_attempt')
                for _ in range(attempt):
                    try:
                        sotimeout = float(pylons.config['download_thread_sotimeout'])
                        proxies = json.loads(pylons.config['urlgrabber_proxies'])
                        urlgrabber.urlgrab(self.__uriDict['uri'], 
                                           self.__uriDict['inProgressPackagePath'],
#                                            checkfunc = None if self.__skipProp else (PackageUtil.validateDownload, (), {}),
                                           progress_obj = DownloadProgress(self),
                                           throttle = float(pylons.config['package_throttle']),
                                           bandwidth = int(pylons.config['package_bandwidth']),
                                           keepalive = 0,
                                           timeout = sotimeout,
                                           proxies = proxies) 
                        break
                    except Exception as exc:
                        msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3))
                        LOG.warning(msg)
                        if _ == attempt-1:
                            raise exc
                        randsleep = randint(5, 10)                
                        time.sleep(randsleep)
                    
            else:
                # oops! only http supported now
                raise AgentException(Errors.DC_UNSUPPORTED_PROTOCOL, 'Only http protocols is supported at the moment')

            self._checkStop()

            if self.__skipProp:
                LOG.info('Skip validating against prop file')
            else:
                if (not PackageUtil.validatePackage(self.__uriDict['inProgressPackagePath'], 
                                                    self.__uriDict['propPath'])):
                    raise AgentException(Errors.DC_FAILED_VALIDATE, 'Package ' + 
                                         self.__uriDict['packagePath'] + ' failed validation')
                os.utime(self.__uriDict['propPath'], None)
                utils.rchmod(self.__uriDict['propPath'], "777", 'no')
            
            LOG.info('Download complete, rename this file %s' % self.__uriDict['packagePath'])
            os.rename(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath'])
            os.utime(self.__uriDict['packagePath'], None)
            utils.rchmod(self.__uriDict['packagePath'], "777", 'no')
            LOG.info("Download complete, updating progress to 100")
            self._updateStatus(progress = 100)

        except AgentException, exc:
            self._updateStatus(httpStatus = 500, progress = 0, error = exc.getCode(), errorMsg = exc.getMsg())
            msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3))
            LOG.error(msg)
            raise exc
示例#6
0
    def executeScript(self):
        """ execute a script from remote location"""
        scriptpath = None

        try:
            # parse the body
            if (not request.body or request.body == ""):
                LOG.error('invalid body found in post command')
                return errorResult(request, response, 10001, 'No body found in post command', controller = self)

            body = json.loads(request.body.encode('ascii', 'ignore'))
            
            scriptloc = body['scriptLocation'] if 'scriptLocation' in body else None
            scriptname = body['scriptName'] if 'scriptName' in body else None
            
            hasSudo = ('sudoUser' in body and body['sudoUser'])
            sudoUser = body['sudoUser'] if ('sudoUser' in body and body['sudoUser'] != 'root') else None
            
            paramobj = body['params'] if 'params' in body else []
            params = paramobj if type(paramobj) == list else paramobj.split()

            LOG.info('%s %s %s %s %s' % (scriptloc, scriptname, hasSudo, sudoUser, params))

            if not scriptloc:
                return errorResult(request, response, Errors.INVALID_REQUEST, 'Script location not found', controller = self)
            
            if not scriptname:
                return errorResult(request, response, Errors.INVALID_REQUEST, 'Script name not found', controller = self)

            scriptpath = os.path.join(self.dataPath(), scriptname)
            LOG.info('scriptpath = %s' % scriptpath)

            utils.runsyscmd('wget %s -O %s' % (scriptloc, scriptpath))

            if not os.path.exists(scriptpath):
                return errorResult(request, response, Errors.FILE_NOT_FOUND_ERROR, 'Failed to get script %s' % scriptpath, controller = self)

            utils.rchmod(scriptpath, '+rx')

            cmd = [scriptpath]
            if hasSudo:
                cmd.insert(0, 'sudo')
                if sudoUser:
                    cmd.insert(1, sudoUser)
                    cmd.insert(1, '-u')

            for param in params:
                cmd.append(param)

            LOG.info('cmd = %s' % cmd)

            appGlobal = config['pylons.app_globals']
            execThread = ExecThread(appGlobal.threadMgr, cmd)
            execThread.setLogLevel('info')
            contextutils.copyJobContexts(self, execThread)
            execThread.start()
            execThread.threadMgrEvent.wait()

            return statusResult(request, response, execThread, controller = self)

        except Exception as excp:
            return errorResult(request, response, error = Errors.UNKNOWN_ERROR,
                               errorMsg = 'Unknown error when executing cmd %s,  %s - %s' %
                               (scriptpath, str(excp), traceback.format_exc(2)),
                               controller = self)
示例#7
0
    def _untarPackages(self, packages, service, untarRootPath, nicelevel=None, pgksNeedSuffix=None, pathSuffix=''):
        """
        untar all the packages

        @params packages - list of dictionary of parsed packages uri
        @throws AgentException
        """
        self._updateProgress(80)
        count = 0
        for pkgDict in packages.itervalues():
            count += 1

            # check if package path exists already
            # if the package is already untarred, move on
            # else create the package path
            pkgName = pkgDict['packageName']
            untarPath = os.path.join(untarRootPath,
                                    pkgName,
                                    '%s%s' % (pkgDict['packageVersion'], pathSuffix if ((pgksNeedSuffix is not None) and pkgName in pgksNeedSuffix) else ''))

            if os.path.exists(untarPath):
                # perhaps another thread has finished extraction, continue with next package
                continue
            
            os.makedirs(untarPath)

            try:
                self.untar(pkgDict['packagePath'], untarPath, nicelevel)

            except AgentException:
                rmrf(untarPath)

            # Note: i. atleast self should have 'rx' so that we can proceed setting 'rx' for group and others
            # if both belong to same group in future, then just self, group should be enough
            # ensure all parent dir of scripts dir have 'rx' so that we can really navigate to scripts dir and execute
            uname = getuserofpath(untarPath)
            chmod(untarPath, '+rx', sudoUser = uname)
            cronusPath = os.path.join(untarPath, 'cronus')
            if os.path.exists(cronusPath):
                uname = getuserofpath(cronusPath)
                chmod(cronusPath, '+rx', sudoUser = uname)
                # now give all scripts 'rx' permission
                scriptsPath = os.path.join(cronusPath, 'scripts')
                if os.path.exists(scriptsPath):
                    uname = getuserofpath(scriptsPath)
                    rchmod(scriptsPath, '+rx', sudoUser = uname)

            # issue #16, now add symlink to .appdata for easy access
            appdata_path = manifestutil.appDataPath(service)
            link_path = os.path.join(untarPath, '.appdata')
            LOG.info('Create .appdata symlink from %s to %s' % (link_path, appdata_path))
            utils.symlink(appdata_path, link_path)

            
            #Running as cronus user when higher privilege service (i.e. not chown all the package into the application user)
            if (not isHigherPrivilegeService(service)):
                uname = configutil.getAppUser()
                uid, gid = utils.getUidGid(uname)
                rchown(untarPath, uid, gid)

            self._updateProgress(calcProgress(80, 99, float(count) / len(packages)))
示例#8
0
    def executeScript(self):
        """ execute a script from remote location"""
        scriptpath = None

        try:
            # parse the body
            if (not request.body or request.body == ""):
                LOG.error('invalid body found in post command')
                return errorResult(request,
                                   response,
                                   10001,
                                   'No body found in post command',
                                   controller=self)

            body = json.loads(request.body)
            scriptloc = body['script-location'].encode(
                'ascii', 'ignore') if 'script-location' in body else None
            scriptname = body['script-name'].encode(
                'ascii', 'ignore') if 'script-name' in body else None
            needsudo = asbool(
                body['need-sudo']) if 'need-sudo' in body else False
            sudotgt = body['sudo-target'].encode(
                'ascii', 'ignore') if 'sudo-target' in body else None
            paramobj = body['params'] if 'params' in body else []
            params = paramobj if type(paramobj) == list else paramobj.split()

            LOG.info('%s %s %s %s %s' %
                     (scriptloc, scriptname, needsudo, sudotgt, params))

            if scriptloc is None or scriptloc == '':
                return errorResult(request,
                                   response,
                                   10003,
                                   'Script location not found',
                                   controller=self)
            if scriptname is None or scriptname == '':
                return errorResult(request,
                                   response,
                                   10003,
                                   'Script name not found',
                                   controller=self)

            scriptpath = os.path.join(self.dataPath(), scriptname)
            LOG.info('scriptpath = %s' % scriptpath)

            os.system('wget %s -O %s' % (scriptloc, scriptpath))

            if scriptpath is None or not os.path.exists(scriptpath):
                return errorResult(request,
                                   response,
                                   10003,
                                   'Failed to get script %s' % scriptpath,
                                   controller=self)

            rchmod(scriptpath, '+rx')

            cmd = [scriptpath]
            if needsudo:
                cmd.insert(0, 'sudo')
                if sudotgt is not None:
                    cmd.insert(1, sudotgt)
                    cmd.insert(1, '-u')

            for param in params:
                param = param.encode('ascii', 'ignore')
                cmd.append(param)

            LOG.info('cmd = %s' % cmd)

            appGlobal = config['pylons.app_globals']
            execThread = ExecThread(appGlobal.threadMgr, cmd)
            execThread.setLogLevel('info')
            execThread.start()
            execThread.threadMgrEvent.wait()

            return statusResult(request, response, execThread, controller=self)

        except Exception as excp:
            return errorResult(
                request,
                response,
                error=Errors.UNKNOWN_ERROR,
                errorMsg='Unknown error when executing cmd %s,  %s - %s' %
                (scriptpath, str(excp), traceback.format_exc(2)),
                controller=self)