Exemple #1
0
 def __init__(self,
              repoid,
              repoConfig,
              cacheDir,
              expireTime,
              maxFileLength=1024 * 1024 * 1024 * 2,
              retryTime=3,
              socketTimeout=5):
     self.repoid = repoid
     self.repoConfig = repoConfig
     self.cacheDir = cacheDir
     self.expireTime = expireTime
     self.repomdLocation = '/repodata/repomd.xml'
     self.repoDataDir = None
     self.packageCacheDir = '/packages/'
     self.fileFetcher = FileFetcher(socketTimeout, retryTime, maxFileLength)
Exemple #2
0
 def __init__(self, repoid, repoConfig, cacheDir, expireTime,
              maxFileLength=1024*1024*1024*2, retryTime=3, socketTimeout=5):
     self.repoid = repoid
     self.repoConfig = repoConfig
     self.cacheDir = cacheDir
     self.expireTime = expireTime
     self.repomdLocation = '/repodata/repomd.xml'
     self.repoDataDir = None
     self.packageCacheDir = '/packages/'
     self.fileFetcher = FileFetcher(socketTimeout, retryTime, maxFileLength)
Exemple #3
0
class RepoCacheHandler:
    def __init__(self,
                 repoid,
                 repoConfig,
                 cacheDir,
                 expireTime,
                 maxFileLength=1024 * 1024 * 1024 * 2,
                 retryTime=3,
                 socketTimeout=5):
        self.repoid = repoid
        self.repoConfig = repoConfig
        self.cacheDir = cacheDir
        self.expireTime = expireTime
        self.repomdLocation = '/repodata/repomd.xml'
        self.repoDataDir = None
        self.packageCacheDir = '/packages/'
        self.fileFetcher = FileFetcher(socketTimeout, retryTime, maxFileLength)

    def getPackage(self, pkg):
        if self._getRepoDataDir() is None:
            return None
        pkgCacheDir = self._getRepoDataDir() + self.packageCacheDir
        preUmask = os.umask(0)
        if not file_util.makeDir(pkgCacheDir):
            os.umask(preUmask)
            Log.cout(Log.ERROR, 'Make cache dir [%s] failed' % pkgCacheDir)
            return None
        os.umask(preUmask)
        uri = pkg.getLocation()
        if not uri.lower().startswith('http'):
            return None
        pkgFilePath = pkgCacheDir + os.path.basename(uri)
        if file_util.exists(pkgFilePath):
            ctime = os.stat(pkgFilePath).st_ctime
            nowtime = time.time()
            if nowtime - ctime < self.expireTime:
                Log.cout(Log.DEBUG, "Get package [%s] from cache" % pkg)
                return pkgFilePath
            if not file_util.remove(pkgFilePath):
                Log.cout(Log.DEBUG,
                         "Remove old package [%s] because of expired" % pkg)
                return None

        if not self.fileFetcher.fetch(uri, pkgFilePath) or \
                not file_util.chmod(pkgFilePath, 0666):
            return None
        return pkgFilePath
Exemple #4
0
class RepoCacheHandler:
    def __init__(self, repoid, repoConfig, cacheDir, expireTime,
                 maxFileLength=1024*1024*1024*2, retryTime=3, socketTimeout=5):
        self.repoid = repoid
        self.repoConfig = repoConfig
        self.cacheDir = cacheDir
        self.expireTime = expireTime
        self.repomdLocation = '/repodata/repomd.xml'
        self.repoDataDir = None
        self.packageCacheDir = '/packages/'
        self.fileFetcher = FileFetcher(socketTimeout, retryTime, maxFileLength)

    def getPackage(self, pkg):
        if self._getRepoDataDir() is None:
            return None
        pkgCacheDir = self._getRepoDataDir() + self.packageCacheDir
        preUmask = os.umask(0)
        if not file_util.makeDir(pkgCacheDir):
            os.umask(preUmask)
            Log.cout(Log.ERROR, 'Make cache dir [%s] failed' % pkgCacheDir)
            return None
        os.umask(preUmask)
        uri = pkg.getLocation()
        if not uri.lower().startswith('http'):
            return None
        pkgFilePath = pkgCacheDir + os.path.basename(uri)
        if file_util.exists(pkgFilePath):
            ctime = os.stat(pkgFilePath).st_ctime
            nowtime = time.time()
            if nowtime - ctime < self.expireTime:
                Log.cout(Log.DEBUG, "Get package [%s] from cache" % pkg)
                return pkgFilePath
            if not file_util.remove(pkgFilePath):
                Log.cout(Log.DEBUG, "Remove old package [%s] because of expired" % pkg)
                return None
                
        if not self.fileFetcher.fetch(uri, pkgFilePath) or \
                not file_util.chmod(pkgFilePath, 0666):
            return None
        return pkgFilePath

    def makeCache(self, metatype=PRIMARY_TYPE, force=False):
        if self._getRepoDataDir() is None:
            return False
        needCache = False
        cacheType = None
        if force is True:
            needCache = True
            cacheType = metatype
        else:
            (needCache, cacheType) = self._needMakeCache(metatype)
        if needCache is False:
            Log.cout(Log.INFO, 'Need not update cache: %s' % self.repoid)
            return True
        ret = False
        Log.cout(Log.INFO, 'Need update cache: %s' % self.repoid)
        if self._doMakeCache(cacheType):
            ret = self._onMakeCacheSuccess(cacheType)
        else:
            self._onMakeCacheFailed(cacheType)
        return ret

    def clearCache(self):
        return file_util.remove(self._getRepoDataDir())

    def getMetaData(self):
        if not self.makeCache(ALL_TYPE):
            Log.cout(Log.ERROR, 'Make cache failed: %s' % self.repoid)
            return None
        if file_util.exists(self._getRepoDataDir() + '/' + PRIMARYDB_FILE):
            Log.cout(Log.INFO, 'Using primatydb format: %s' % self.repoid)
            return sqlite_loader.loadFrom(self._getRepoDataDir() +
                                          '/' + PRIMARYDB_FILE)
        elif file_util.exists(self._getRepoDataDir() + '/' + PRIMARY_FILE):
            Log.cout(Log.INFO, 'Using primaty format: %s' % self.repoid)
            return cache.loadFromFile(self._getRepoDataDir() +
                                      '/' + PRIMARY_FILE)
        Log.cout(Log.ERROR, 'repodb not found: %s' % self.repoid)
        return None
        

    def _filterMetaTypeByRepoMD(self, metatype, repoMdObj):
        hasPrimary = repoMdObj.repoMdDatas.has_key(PRIMARY_NAME)
        hasPrimaryDB = repoMdObj.repoMdDatas.has_key(PRIMARYDB_NAME)
        if metatype == ALL_TYPE:
            if not hasPrimaryDB:
                return PRIMARY_TYPE
            elif not hasPrimary:
                return PRIMARYDB_TYPE
        return metatype
    
    def _doMakeCache(self, metatype):
        #download repomd.xml to repomd.xml.tmp
        repomdTmpFile = self._getRepoDataDir() + '/' + \
            self.repomdLocation.split('/')[-1] + ".tmp"
        repomdUrl = self.repoConfig.baseurl + self.repomdLocation
        if not file_util.remove(repomdTmpFile) or \
                not self.fileFetcher.fetch(repomdUrl, repomdTmpFile) or\
                not file_util.chmod(repomdTmpFile, 0666):
            return False
        repoMdObj = RepoMdParser().parse(repomdTmpFile)
        if not repoMdObj:
            return False
        metatype = self._filterMetaTypeByRepoMD(metatype, repoMdObj)
        if metatype == PRIMARY_TYPE:
            return self._getMetaFile(repoMdObj, PRIMARY_NAME, PRIMARY_FILE)
        elif metatype == PRIMARYDB_TYPE:
            return self._getMetaFile(repoMdObj, PRIMARYDB_NAME, PRIMARYDB_FILE)
        elif metatype == ALL_TYPE:
            return self._getMetaFile(repoMdObj, PRIMARY_NAME, PRIMARY_FILE) and \
                self._getMetaFile(repoMdObj, PRIMARYDB_NAME, PRIMARYDB_FILE)
        else:
            return False

    def _getMetaFile(self, repoMdObj, metaName, fileName):
        if not repoMdObj.repoMdDatas.has_key(metaName):
            return False
        metaObj = repoMdObj.repoMdDatas[metaName]
        destTmpFile = self._getRepoDataDir() + '/' + \
            metaObj.locationHref.split('/')[-1]
        metaUrl = self.repoConfig.baseurl + '/' + metaObj.locationHref
#        uncompressTmpFile = '.'.join(destTmpFile.split('.')[:-1]) + '.tmp'
        uncompressTmpFile = self._getRepoDataDir() + '/' + \
            fileName + '.tmp'
        if not file_util.remove(destTmpFile) or\
                not file_util.remove(uncompressTmpFile):
            return False
        if not self.fileFetcher.fetch(metaUrl, destTmpFile) or\
                not file_util.chmod(destTmpFile, 0666):
            return False
        try:
            if destTmpFile.split('.')[-1] == 'bz2':
                f = bz2.BZ2File(destTmpFile)
            else:
                f = gzip.open(destTmpFile)
            if not file_util.writeToFile(uncompressTmpFile, f.read()) or\
                    not file_util.chmod(uncompressTmpFile, 0666):
                f.close()
                return False
            f.close()
        except Exception:
            Log.cout(Log.ERROR, 'decompress %s failed' % destTmpFile)
            return False
        return self._checkSumValid(metaObj, uncompressTmpFile)
    
    def _needMakeCache(self, metatype):
        #check cachecookie
        cachecookie = self._getRepoDataDir() + '/' + 'cachecookie';
        if not file_util.exists(cachecookie):
            return (True, metatype)
        ctime = os.stat(cachecookie).st_ctime
        nowtime = time.time()
        if nowtime - ctime > self.expireTime:
            return (True, metatype)
        #check repomd.xml
        repomdFile = self._getRepoDataDir() + '/' + self.repomdLocation.split('/')[-1]
        if not file_util.exists(repomdFile):
            return (True, metatype)
        mdParser = RepoMdParser()
        repoMd = mdParser.parse(repomdFile)
        if repoMd is None:
            return (True, metatype)
        metatype = self._filterMetaTypeByRepoMD(metatype, repoMd)
        #check metatype
        if metatype == PRIMARY_TYPE:
            if not self._checkMetaFile(repoMd, PRIMARY_NAME, PRIMARY_FILE):
                return (True, metatype)
        elif metatype == PRIMARYDB_TYPE:
            if not self._checkMetaFile(repoMd, PRIMARYDB_NAME, PRIMARYDB_FILE):
                return (True, metatype)
        elif metatype == ALL_TYPE:
            primaryRet = self._checkMetaFile(repoMd, PRIMARY_NAME, PRIMARY_FILE)
            primarydbRet = self._checkMetaFile(repoMd,
                                               PRIMARYDB_NAME, PRIMARYDB_FILE)
            if primaryRet and not primarydbRet:
                return (True, PRIMARYDB_TYPE)
            elif not primaryRet and primarydbRet:
                return (True, PRIMARY_TYPE)
            elif not primaryRet and not primarydbRet:
                return (True, ALL_TYPE)
        return (False, None)

    def _checkMetaFile(self, repoMd, metaName, fileName):
        metaFile = self._getRepoDataDir() + '/' + fileName
        if not file_util.exists(metaFile):
            return False
        if not repoMd.repoMdDatas.has_key(metaName):
            Log.cout(Log.ERROR, '%s not found in repomd.xml' % metaName)
            return False
        metaObj = repoMd.repoMdDatas[metaName]
        return self._checkSumValid(metaObj, metaFile)

    def _onMakeCacheSuccess(self, cacheType):
        cachecookie = self._getRepoDataDir() + '/' + 'cachecookie';
        if not file_util.remove(cachecookie) or\
                not file_util.writeToFile(cachecookie, '') or\
                not file_util.chmod(cachecookie, 0666):
            Log.cout(Log.ERROR, 'Re-touch cachecookie failed')
            return False
        repomdTmpFile = self._getRepoDataDir() + '/' + \
            self.repomdLocation.split('/')[-1];
        if not file_util.rename(repomdTmpFile + '.tmp', repomdTmpFile):
            return False
        return self._processTmpFile(cacheType, True)

    def _onMakeCacheFailed(self, cacheType):
        repomdTmpFile = self._getRepoDataDir() + '/' + \
            self.repomdLocation.split('/')[-1];
        if not file_util.remove(repomdTmpFile + '.tmp'):
            return False
        return self._processTmpFile(cacheType, False)

    def _processTmpFile(self, cacheType, success):
        nameList = []
        if cacheType == PRIMARY_TYPE:
            nameList.append(PRIMARY_FILE)
        elif cacheType == PRIMARYDB_TYPE:
            nameList.append(PRIMARYDB_FILE)
        elif cacheType == ALL_TYPE:
            nameList.append(PRIMARY_FILE)
            nameList.append(PRIMARYDB_FILE)
        for name in nameList:
            fileName = self._getRepoDataDir() + '/' + name
            if not file_util.exists(fileName + '.tmp'):
                continue
            if success:
                if not file_util.rename(fileName + '.tmp', fileName):
                    return False
            else:
                if not file_util.remove(fileName + '.tmp'):
                    return False
            if not file_util.remove(fileName + '.gz'):
                return False
        return True

    def _getRepoDataDir(self):
        if self.repoDataDir:
            return self.repoDataDir
        if not self.cacheDir or not self.repoid:
            return None
        self.repoDataDir = self.cacheDir + '/' + self.repoid
        if not file_util.exists(self.repoDataDir):
            preUmask = os.umask(0)
            ret = file_util.makeDir(self.repoDataDir, True, 0777)
            os.umask(preUmask)
            if not ret: 
                Log.cout(Log.ERROR, 'make repo data dir %s failed' % self.repoDataDir)
                return None
        return self.repoDataDir

    def _checkSumValid(self, metaObj, filePath):
        if metaObj.openchecksumType == "sha":
            return  self._sha1(filePath) == metaObj.openchecksumValue
        elif metaObj.openchecksumType == "md5":
            return  self._md5(filePath) == metaObj.openchecksumValue
        return False

    def _md5(self, filePath):
        f = open(filePath,'rb')
        md5Obj = md5.new()
        md5Obj.update(f.read())
        f.close()
        return md5Obj.hexdigest()
        
    def _sha1(self, filePath):
        f = open(filePath,'rb')
        shaObj = sha.new()
        shaObj.update(f.read())
        f.close()
        return shaObj.hexdigest()