def getDbFile(self, dbtype): if dbtype != "primary": log.info2("Loading %s for %s...", dbtype, self.reponame) cachebase = os.path.join(self.config.cachedir, self.reponame) cachepath = os.path.join(self.config.cachedir, self.reponame, "sqlite") if not os.path.isdir(cachebase): os.makedirs(cachebase) if not os.path.isdir(cachepath): os.makedirs(cachepath) # check existing sqlite db dbfilename = os.path.join(cachepath, "%s.xml.gz.sqlite" % dbtype) if os.path.exists(dbfilename): try: csum, db = self.loadCache(dbfilename) except sqlite3.Error, e: log.error(e) csum = None if self.repomd.has_key(dbtype) and \ self.repomd[dbtype].has_key("checksum") and \ csum == self.repomd[dbtype]["checksum"]: setattr(self, "_%sdb" % dbtype, db) setattr(self, "_%sdb_cursor" % dbtype, db.cursor()) return 1
def loadCache(self, filename): """Load cache from filename, check if it is valid and that dbversion matches the required dbversion""" db = sqlite3.connect(filename) db.row_factory = sqlite3.Row db.text_factory = str cur = db.cursor() cur.execute("SELECT * FROM db_info") info = cur.fetchone() # If info is not in there this is an incompelete cache file # (this could happen when the user hits ctrl-c or kills yum # when the cache is being generated or updated) if not info: raise sqlite3.DatabaseError, "Incomplete database cache file" # Now check the database version if info['dbversion'] not in supported_dbversions: log.info2( "Cache file is version %s, we need %s, will " "regenerate.\n", info['dbversion'], dbversion) raise sqlite3.DatabaseError, "Older version of yum sqlite: %s" % info[ 'dbversion'] # This appears to be a valid database, return checksum value and # database object return (info['checksum'], db)
def __readDir(self, dir, location): """Look for non-excluded *.rpm files under dir and add them to self.pkglist. dir must be a local file system path. The remote location prefix corresponding to dir is location. Set pkg["yumlocation"] to the remote relative path to the package.""" tmplist = [] functions.readDir(dir, tmplist, ("name", "epoch", "version", "release", "arch", "sourcerpm", "requirename", "requireflags", "requireversion")) for pkg in tmplist: # FIXME: this is done in createRepo too # If it is a source rpm change the arch to "src". Only valid # for createRepo, never do this anywhere else. ;) if pkg.isSourceRPM(): pkg["arch"] = "src" nevra = pkg.getNEVRA() log.info2("Adding %s to repo and checking file requires.", nevra) pkg["yumlocation"] = location+pkg.source[len(dir):] self.addPkg(pkg)
def run_main(main): """Run main, handling --hotshot. The return value from main, if not None, is a return code.""" dohotshot = 0 if len(sys.argv) >= 2 and sys.argv[1] == "--hotshot": dohotshot = 1 sys.argv.pop(1) if dohotshot: import hotshot, hotshot.stats htfilename = mkstemp_file("/tmp", tmpprefix)[1] prof = hotshot.Profile(htfilename) prof.runcall(main) prof.close() del prof log.info2("Starting profil statistics. This takes some time...") s = hotshot.stats.load(htfilename) s.strip_dirs().sort_stats("time").print_stats(100) s.strip_dirs().sort_stats("cumulative").print_stats(100) os.unlink(htfilename) else: return main()
def __readDir(self, dir, location): """Look for non-excluded *.rpm files under dir and add them to self.pkglist. dir must be a local file system path. The remote location prefix corresponding to dir is location. Set pkg["yumlocation"] to the remote relative path to the package.""" tmplist = [] functions.readDir( dir, tmplist, ("name", "epoch", "version", "release", "arch", "sourcerpm", "requirename", "requireflags", "requireversion")) for pkg in tmplist: # FIXME: this is done in createRepo too # If it is a source rpm change the arch to "src". Only valid # for createRepo, never do this anywhere else. ;) if pkg.isSourceRPM(): pkg["arch"] = "src" nevra = pkg.getNEVRA() log.info2("Adding %s to repo and checking file requires.", nevra) pkg["yumlocation"] = location + pkg.source[len(dir):] self.addPkg(pkg)
def loadCache(self, filename): """Load cache from filename, check if it is valid and that dbversion matches the required dbversion""" db = sqlite3.connect(filename) db.row_factory = sqlite3.Row db.text_factory = str cur = db.cursor() cur.execute("SELECT * FROM db_info") info = cur.fetchone() # If info is not in there this is an incompelete cache file # (this could happen when the user hits ctrl-c or kills yum # when the cache is being generated or updated) if not info: raise sqlite3.DatabaseError, "Incomplete database cache file" # Now check the database version if info['dbversion'] not in supported_dbversions: log.info2("Cache file is version %s, we need %s, will " "regenerate.\n", info['dbversion'], dbversion) raise sqlite3.DatabaseError, "Older version of yum sqlite: %s" % info['dbversion'] # This appears to be a valid database, return checksum value and # database object return (info['checksum'], db)
# ofd = gzip.GzipFile(os.path.join(datapath, "other.xml.gz"), "wb") #except IOError: # return 0 pdoc = libxml2.newDoc("1.0") proot = pdoc.newChild(None, "metadata", None) fdoc = libxml2.newDoc("1.0") froot = fdoc.newChild(None, "filelists", None) #odoc = libxml2.newDoc("1.0") #oroot = odoc.newChild(None, "filelists", None) log.info1("Pass 2: Writing repodata information.") pfd.write('<?xml version="1.0" encoding="UTF-8"?>\n') pfd.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%d">\n' % len(self.getPkgs())) ffd.write('<?xml version="1.0" encoding="UTF-8"?>\n') ffd.write('<filelists xmlns:rpm="http://linux.duke.edu/filelists" packages="%d">\n' % len(self.getPkgs())) for pkg in self.getPkgs(): log.info2("Processing complete data of package %s.", pkg.getNEVRA()) pkg.header_read = 0 try: pkg.open() pkg.read() except (IOError, ValueError), e: log.warning("%s: %s", pkg.getNEVRA(), e) continue # If it is a source rpm change the arch to "src". Only valid # for createRepo, never do this anywhere else. ;) if pkg.isSourceRPM(): pkg["arch"] = "src" try: checksum = self.__getChecksum(pkg) except (IOError, NotImplementedError), e: log.warning("%s: %s", pkg.getNEVRA(), e)
self.repomd[dbtype].has_key("checksum") and \ csum == self.repomd[dbtype]["checksum"] and \ self.nc.isCached("repodata/%s.xml.gz" % dbtype): filename = self.nc.getCachedFilename("repodata/%s.xml.gz" % dbtype) else: filename = self.nc.cache("repodata/%s.xml.gz" % dbtype, 1, copy_local=True) (csum, destfile) = self.nc.checksum("repodata/%s.xml.gz" % dbtype, "sha") if not (self.repomd.has_key(dbtype) and \ self.repomd[dbtype].has_key("checksum") and \ csum == self.repomd[dbtype]["checksum"]): return 0 if filename: log.info2("Creating %s", dbfilename) if USEYUM: parser = yum.mdparser.MDParser(filename) storage = yum.storagefactory.GetStorage() cache = storage.GetCacheHandler(dbfilename, 'tmp', None) if dbtype == 'primary': db = cache.getPrimary(filename, csum) elif dbtype == 'filelists': db = cache.getFilelists(filename, csum) elif dbtype == 'other': db = cache.getOtherdata(filename, csum) # XXX error handling shutil.move(filename + '.sqlite', dbfilename) setattr(self, "_%sdb" % dbtype, db) setattr(self, "_%sdb_cursor" % dbtype, db.cursor()) # TODO: add all other indices
csum == self.repomd[dbtype]["checksum"] and \ self.nc.isCached("repodata/%s.xml.gz" % dbtype): filename = self.nc.getCachedFilename("repodata/%s.xml.gz" % dbtype) else: filename = self.nc.cache("repodata/%s.xml.gz" % dbtype, 1, copy_local=True) (csum, destfile) = self.nc.checksum("repodata/%s.xml.gz" % dbtype, "sha") if not (self.repomd.has_key(dbtype) and \ self.repomd[dbtype].has_key("checksum") and \ csum == self.repomd[dbtype]["checksum"]): return 0 if filename: log.info2("Creating %s", dbfilename) if USEYUM: parser = yum.mdparser.MDParser(filename) storage = yum.storagefactory.GetStorage() cache = storage.GetCacheHandler(dbfilename, 'tmp', None) if dbtype == 'primary': db = cache.getPrimary(filename, csum) elif dbtype == 'filelists': db = cache.getFilelists(filename, csum) elif dbtype == 'other': db = cache.getOtherdata(filename, csum) # XXX error handling shutil.move(filename + '.sqlite', dbfilename) setattr(self, "_%sdb" % dbtype, db) setattr(self, "_%sdb_cursor" % dbtype, db.cursor()) # TODO: add all other indices
def read(self): log.info2("Reading RHN channel repository '%s'", self.reponame) return SqliteRepoDB.read(self)
proot = pdoc.newChild(None, "metadata", None) fdoc = libxml2.newDoc("1.0") froot = fdoc.newChild(None, "filelists", None) #odoc = libxml2.newDoc("1.0") #oroot = odoc.newChild(None, "filelists", None) log.info1("Pass 2: Writing repodata information.") pfd.write('<?xml version="1.0" encoding="UTF-8"?>\n') pfd.write( '<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%d">\n' % len(self.getPkgs())) ffd.write('<?xml version="1.0" encoding="UTF-8"?>\n') ffd.write( '<filelists xmlns:rpm="http://linux.duke.edu/filelists" packages="%d">\n' % len(self.getPkgs())) for pkg in self.getPkgs(): log.info2("Processing complete data of package %s.", pkg.getNEVRA()) pkg.header_read = 0 try: pkg.open() pkg.read() except (IOError, ValueError), e: log.warning("%s: %s", pkg.getNEVRA(), e) continue # If it is a source rpm change the arch to "src". Only valid # for createRepo, never do this anywhere else. ;) if pkg.isSourceRPM(): pkg["arch"] = "src" try: checksum = self.__getChecksum(pkg) except (IOError, NotImplementedError), e: log.warning("%s: %s", pkg.getNEVRA(), e)