コード例 #1
0
    def doRun(self):
        """ Main body of the thread """
        try:
            path = pylons.config['repo_root']
            packagePath = os.path.join(path, self.__package)
            propPath = os.path.join(path, (self.__package + '.prop'))
            if os.path.exists(packagePath):
                if (os.path.exists(propPath)
                        and PackageUtil.validateProp(propPath)
                        and PackageUtil.validatePackage(packagePath)):
                    os.utime(packagePath, None)
                    os.utime(propPath, None)
                    self._updateStatus(
                        progress=100,
                        result={'msg': 'Package %s is valid' % packagePath})

                else:
                    msg = 'Package %s failed validation ' % packagePath
                    LOG.warning(msg)
                    self._updateStatus(httpStatus=500,
                                       progress=100,
                                       error=Errors.PACKAGE_CHECKSUM_ERROR,
                                       errorMsg=msg)
            else:
                self._updateStatus(httpStatus=500,
                                   progress=100,
                                   error=Errors.PACKAGE_NOT_FOUND,
                                   errorMsg=('Package not found %s' %
                                             packagePath))

        except Exception as excp:
            errorMsg = 'Exception downloading %s - traceback %s' % (
                str(excp), traceback.format_exc(2))
            self._updateStatus(httpStatus=500,
                               progress=100,
                               error=Errors.UNKNOWN_ERROR,
                               errorMsg=errorMsg)
コード例 #2
0
 def doRun(self):
     """ Main body of the thread """
     try:
         path = pylons.config['repo_root']
         packagePath = os.path.join(path, self.__package)
         propPath = os.path.join(path, (self.__package + '.prop'))
         if os.path.exists(packagePath):
             if (os.path.exists(propPath) and
                 PackageUtil.validateProp(propPath) and
                 PackageUtil.validatePackage(packagePath)):
                 os.utime(packagePath, None)
                 os.utime(propPath, None)
                 self._updateStatus(progress = 100, result = {'msg': 'Package %s is valid' % packagePath})
                 
             else:    
                 msg = 'Package %s failed validation ' % packagePath
                 LOG.warning(msg)
                 self._updateStatus(httpStatus = 500, progress = 100, error = Errors.PACKAGE_CHECKSUM_ERROR, errorMsg = msg)
         else:
             self._updateStatus(httpStatus = 500, progress = 100, error = Errors.PACKAGE_NOT_FOUND, errorMsg = ('Package not found %s' % packagePath))
             
     except Exception as excp:
         errorMsg = 'Exception downloading %s - traceback %s' % (str(excp), traceback.format_exc(2))
         self._updateStatus(httpStatus = 500, progress = 100, error = Errors.UNKNOWN_ERROR, errorMsg = errorMsg)
コード例 #3
0
    def __startDownload(self):
        """ actual download logic """
        try:
            LOG.info("Starting package download for package %s" %
                     self.__uriDict['package'])

            # check to see if there's an in progress file,
            # since PackageMgr guarantees that duplicate threads will not be spawned
            # for same pkg, assume an existing thread was killed.
            # attempt to clean up package n move
            if (os.path.exists(self.__uriDict['inProgressPackagePath'])):
                LOG.debug(
                    'In progress file (%s) already exists. Will validate and reattempt download if necessary'
                    % self.__uriDict['inProgressPackagePath'])

            if os.path.exists(self.__uriDict['packagePath']):
                if (os.path.exists(self.__uriDict['propPath']) and
                        PackageUtil.validateProp(self.__uriDict['propPath'])
                        and PackageUtil.validatePackage(
                            self.__uriDict['packagePath'],
                            self.__uriDict['propPath'])):
                    msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict[
                        'packagePath']
                    LOG.info(msg)
                    os.utime(self.__uriDict['packagePath'], None)
                    os.utime(self.__uriDict['propPath'], None)
                    self._updateStatus(progress=100)
                    # NOTE: this is a normal exit not an error!
                    return
                LOG.warning(
                    'The package already exists. However package prop (%s) failed validation. Downloading package.'
                    % self.__uriDict['propPath'])

            # Delete all traces of package before beginning download
            LOG.info('Cleaning up all packages for %s ' %
                     self.__uriDict['packagePath'])
            PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'],
                                       self.__uriDict['packagePath'],
                                       self.__uriDict['propPath'])

            AgentThread._updateProgress(self, 0)

            if not self.__skipProp:
                # First, download .prop file
                LOG.info(
                    'Starting download of prop file %s - %s' %
                    (self.__uriDict['propUri'], self.__uriDict['propPath']))
                self.__download_prop_file()
                try:
                    self.__prop = loadPropFile(self.__uriDict['propPath'])
                except FileNotFoundError:
                    raise AgentException(
                        Errors.DC_MISSING_PROP_FILE,
                        'Prop file (%s) unable to read or did not parse' %
                        (self.__uriDict['propPath']))
            AgentThread._updateProgress(self, 2)

            self.__setProgressTimeouts()

            if self.__uriDict['scheme'] == 'http':
                # try download 3 times, with random sleep
                for _ in range(3):
                    try:
                        sotimeout = float(
                            pylons.config['download_thread_sotimeout'])
                        proxies = json.loads(
                            pylons.config['urlgrabber_proxies'])
                        urlgrabber.urlgrab(
                            self.__uriDict['uri'],
                            self.__uriDict['inProgressPackagePath'],
                            checkfunc=None if self.__skipProp else
                            (PackageUtil.validateDownload, (), {}),
                            progress_obj=DownloadProgress(self),
                            throttle=float(pylons.config['package_throttle']),
                            bandwidth=int(pylons.config['package_bandwidth']),
                            keepalive=0,
                            timeout=sotimeout,
                            proxies=proxies)
                        break
                    except Exception as exc:
                        msg = 'Download error %s - %s' % (
                            str(exc), traceback.format_exc(3))
                        LOG.warning(msg)
                        randsleep = randint(30, 60)
                        time.sleep(randsleep)

            else:
                # oops! only http and bittorrent supported now
                raise AgentException(
                    Errors.DC_UNSUPPORTED_PROTOCOL,
                    'Only http protocols is supported at the moment')

            self._checkStop()

            if not self.__skipProp:
                if (not PackageUtil.validatePackage(
                        self.__uriDict['inProgressPackagePath'],
                        self.__uriDict['propPath'])):
                    raise AgentException(
                        Errors.DC_FAILED_VALIDATE, 'Package ' +
                        self.__uriDict['packagePath'] + ' failed validation')
                os.utime(self.__uriDict['propPath'], None)
                utils.rchmod(self.__uriDict['propPath'], "777", 'no')

            LOG.info(
                'Download complete, will now rename and do validation on this file %s'
                % self.__uriDict['packagePath'])
            os.rename(self.__uriDict['inProgressPackagePath'],
                      self.__uriDict['packagePath'])
            os.utime(self.__uriDict['packagePath'], None)
            utils.rchmod(self.__uriDict['packagePath'], "777", 'no')
            LOG.info(
                "Download complete, Validation completed, updating progress to 100"
            )
            self._updateStatus(progress=100)

        except AgentException, exc:
            self._updateStatus(httpStatus=500,
                               progress=0,
                               error=exc.getCode(),
                               errorMsg=exc.getMsg())
            msg = 'Download error %s - %s' % (str(exc),
                                              traceback.format_exc(3))
            LOG.error(msg)
            raise exc
コード例 #4
0
    def __startDownload(self):
        """ actual download logic """
        try:
            LOG.info("Starting package download for package %s" % self.__uriDict['package'])

            # check to see if there's an in progress file,
            # since PackageMgr guarantees that duplicate threads will not be spawned
            # for same pkg, assume an existing thread was killed.
            # attempt to clean up package n move
            if (os.path.exists(self.__uriDict['inProgressPackagePath'])):
                LOG.debug('In progress file (%s) already exists. Cleanup and reattempt download' 
                          % self.__uriDict['inProgressPackagePath'])


            if os.path.exists(self.__uriDict['packagePath']):
                if self.__skipProp or ((os.path.exists(self.__uriDict['propPath']) and
                                        PackageUtil.validateProp(self.__uriDict['propPath']) and
                                        PackageUtil.validatePackage(self.__uriDict['packagePath'], 
                                                                    self.__uriDict['propPath']))):
                    msg = 'The package already exists. Will NOT download duplicate package' + self.__uriDict['packagePath']
                    LOG.info(msg)
                    os.utime(self.__uriDict['packagePath'], None)
                    if os.path.exists(self.__uriDict['propPath']):
                        os.utime(self.__uriDict['propPath'], None)
                    self._updateStatus(progress = 100)
                    # NOTE: this is a normal exit not an error!
                    return
                else:
                    LOG.warning('The package already exists. However package prop (%s) failed validation.' 
                                % self.__uriDict['propPath'])

            # Delete all traces of package before beginning download
            LOG.debug('Cleaning up all packages for %s ' % self.__uriDict['packagePath'])
            PackageUtil.cleanUpPackage(self.__uriDict['inProgressPackagePath'],
                                   self.__uriDict['packagePath'],
                                   self.__uriDict['propPath'])

            AgentThread._updateProgress(self, 0)
            
            if self.__skipProp:
                LOG.info('Skip download of prop file')
            else:
                # First, download .prop file
                LOG.info('Starting download of prop file %s - %s' % (self.__uriDict['propUri'], self.__uriDict['propPath']))
                self.__download_prop_file()
                try:
                    self.__prop = loadPropFile(self.__uriDict['propPath'])
                except FileNotFoundError:
                    raise AgentException(Errors.DC_MISSING_PROP_FILE,
                                         'Prop file (%s) unable to read or did not parse' % (self.__uriDict['propPath']))
                    
            AgentThread._updateProgress(self, 2)
            self.__setProgressTimeouts()

            if self.__uriDict['scheme'] == 'http':
                # try download 3 times, with random sleep
                attempt = configutil.getConfigAsInt('download_thread_attempt')
                for _ in range(attempt):
                    try:
                        sotimeout = float(pylons.config['download_thread_sotimeout'])
                        proxies = json.loads(pylons.config['urlgrabber_proxies'])
                        urlgrabber.urlgrab(self.__uriDict['uri'], 
                                           self.__uriDict['inProgressPackagePath'],
#                                            checkfunc = None if self.__skipProp else (PackageUtil.validateDownload, (), {}),
                                           progress_obj = DownloadProgress(self),
                                           throttle = float(pylons.config['package_throttle']),
                                           bandwidth = int(pylons.config['package_bandwidth']),
                                           keepalive = 0,
                                           timeout = sotimeout,
                                           proxies = proxies) 
                        break
                    except Exception as exc:
                        msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3))
                        LOG.warning(msg)
                        if _ == attempt-1:
                            raise exc
                        randsleep = randint(5, 10)                
                        time.sleep(randsleep)
                    
            else:
                # oops! only http supported now
                raise AgentException(Errors.DC_UNSUPPORTED_PROTOCOL, 'Only http protocols is supported at the moment')

            self._checkStop()

            if self.__skipProp:
                LOG.info('Skip validating against prop file')
            else:
                if (not PackageUtil.validatePackage(self.__uriDict['inProgressPackagePath'], 
                                                    self.__uriDict['propPath'])):
                    raise AgentException(Errors.DC_FAILED_VALIDATE, 'Package ' + 
                                         self.__uriDict['packagePath'] + ' failed validation')
                os.utime(self.__uriDict['propPath'], None)
                utils.rchmod(self.__uriDict['propPath'], "777", 'no')
            
            LOG.info('Download complete, rename this file %s' % self.__uriDict['packagePath'])
            os.rename(self.__uriDict['inProgressPackagePath'], self.__uriDict['packagePath'])
            os.utime(self.__uriDict['packagePath'], None)
            utils.rchmod(self.__uriDict['packagePath'], "777", 'no')
            LOG.info("Download complete, updating progress to 100")
            self._updateStatus(progress = 100)

        except AgentException, exc:
            self._updateStatus(httpStatus = 500, progress = 0, error = exc.getCode(), errorMsg = exc.getMsg())
            msg = 'Download error %s - %s' % (str(exc), traceback.format_exc(3))
            LOG.error(msg)
            raise exc