class RpmRepoDB(memorydb.RpmMemoryDB): """A (mostly) read-only RPM database storage in repodata XML. This is not a full implementation of Database: notably the file database is not populated at all.""" # A mapping between strings and RPMSENSE_* comparison flags flagmap = { 0 : None, None: 0, "EQ": RPMSENSE_EQUAL, "LT": RPMSENSE_LESS, "GT": RPMSENSE_GREATER, "LE": RPMSENSE_EQUAL | RPMSENSE_LESS, "GE": RPMSENSE_EQUAL | RPMSENSE_GREATER, RPMSENSE_EQUAL: "EQ", RPMSENSE_LESS: "LT", RPMSENSE_GREATER: "GT", RPMSENSE_EQUAL | RPMSENSE_LESS: "LE", RPMSENSE_EQUAL | RPMSENSE_GREATER: "GE"} def __init__(self, config, source, buildroot='', reponame="default", nc=None): """Exclude packages matching whitespace-separated excludes. Use reponame for cache subdirectory name and pkg["yumreponame"]. Load PGP keys from URLs in key_urls.""" memorydb.RpmMemoryDB.__init__(self, config, source, buildroot) self.reponame = reponame self.excludes = self.config.excludes[:] self.mirrorlist = None self.baseurls = None self.yumconf = None self.key_urls = [] if nc: self.nc = nc else: self.nc = NetworkCache([], self.config.cachedir, self.reponame) if isinstance(source, types.DictType): found_urls = False self.yumconf = source if self.yumconf.has_key("main"): sec = self.yumconf["main"] if sec.has_key("exclude"): self.excludes.extend(sec["exclude"]) sec = self.yumconf[self.reponame] if sec.has_key("exclude"): self.excludes.extend(sec["exclude"]) if sec.has_key("gpgkey"): self.key_urls = sec["gpgkey"] if sec.has_key("baseurl"): self.nc.addCache(sec["baseurl"], self.reponame) found_urls = True if sec.has_key("mirrorlist"): self.mirrorlist = sec["mirrorlist"] found_urls = True if not found_urls: raise ValueError, "yum.conf is missing mirrorlist or baseurl parameter" else: self.baseurls = source self.nc.addCache(self.baseurls, self.reponame) self.repomd = None self.filelist_imported = 0 # Files included in primary.xml self._filerc = re.compile('^(.*bin/.*|/etc/.*|/usr/lib/sendmail)$') self._dirrc = re.compile('^(.*bin/.*|/etc/.*)$') self.comps = None def readMirrorList(self): if not self.is_read and self.mirrorlist and self.yumconf: fname = self.nc.cache(self.mirrorlist, 1) if fname: lines = open(fname).readlines() os.unlink(fname) else: lines = [] for l in lines: l = l.strip() l = l.replace("$ARCH", "$basearch") l = self.yumconf.replaceVars(l) if l and l[0] != "#": self.nc.addCache([l,]) def getExcludes(self): return self.excludes def getMirrorList(self): return self.mirrorlist def isIdentitySave(self): """return if package objects that are added are in the db afterwards (.__contains__() returns True and the object are return from searches) """ return False def readRepoMD(self): # First we try and read the repomd file as a starting point. filename = self.nc.cache("repodata/repomd.xml", 1) if not filename: log.error("Couldn't open repomd.xml") return 0 try: fd = open(filename) ip = iterparse(fd, events=("start","end")) ip = iter(ip) except IOError: log.error("Couldn't parse repomd.xml") return 0 # Create our network cache object self.repomd = self._parse(ip) return 1 def readComps(self): # Try to read a comps.xml file if there is any before we parse the # primary.xml if self.repomd.has_key("group"): if not self.repomd["group"].has_key("location"): log.error("Couldn't find proper location for comps.xml in repomd") return 0 comps = self.repomd["group"]["location"] (csum, destfile) = self.nc.checksum(comps, "sha") if self.repomd["group"].has_key("checksum") and \ csum == self.repomd["group"]["checksum"]: filename = destfile else: filename = self.nc.cache(comps, 1) if not filename: return 0 try: self.comps = RpmCompsXML(self.config, filename) self.comps.read() except IOError: return 0 return 1 def readPrimary(self): # If we have either a local cache of the primary.xml.gz file or if # it is already local (nfs or local file system) we calculate it's # checksum and compare it with the one from repomd. If they are # the same we don't need to cache it again and can directly use it. if self.repomd.has_key("primary"): if not self.repomd["primary"].has_key("location"): return 0 primary = self.repomd["primary"]["location"] (csum, destfile) = self.nc.checksum(primary, "sha") if self.repomd["primary"].has_key("checksum") and \ csum == self.repomd["primary"]["checksum"]: filename = destfile else: filename = self.nc.cache(primary, 1) if not filename: return 0 try: fd = PyGZIP(filename) ip = iterparse(fd, events=("start","end")) ip = iter(ip) except IOError: log.error("Couldn't parse primary.xml") return 0 self._parse(ip) return 1 def readPGPKeys(self): for url in self.key_urls: filename = self.nc.cache(url, 1) try: f = file(filename) key_data = f.read() f.close() except Exception, e: log.error("Error reading GPG key %s: %s", filename, e) continue try: key_data = openpgp.isolateASCIIArmor(key_data) keys = openpgp.parsePGPKeys(key_data) except Exception, e: log.error("Invalid GPG key %s: %s", url, e) continue for k in keys: self.keyring.addKey(k)
class Source: """ Load Source repo and extra repos according to kickstart configuration. """ def __init__(self): self.repos = {} self.base_repo_names = [] self.mounts = {} def load(self, ks, dir, beta_key_verify=False): self.dir = dir self.exclude = None # mount source to dir if ks.has_key("cdrom"): self.url = mount_cdrom(dir) if ks["cdrom"].has_key("exclude"): self.exclude = ks["cdrom"]["exclude"] elif ks.has_key("nfs"): opts = None if ks["nfs"].has_key("opts"): opts = ks["nfs"]["opts"] self.url = mount_nfs("nfs://%s:%s" % \ (ks["nfs"]["server"], ks["nfs"]["dir"]), dir, options=opts) if ks["nfs"].has_key("exclude"): self.exclude = ks["nfs"]["exclude"] else: self.url = ks["url"]["url"] if ks["url"].has_key("exclude"): self.exclude = ks["url"]["exclude"] # create network cache self.cache = NetworkCache([self.url], cachedir=rpmconfig.cachedir) # get source information via .discinfo file if not self.cache.cache(".discinfo"): log.error("No .discinfo for '%s'", self.url) return 0 di = get_discinfo(self.cache.cache(".discinfo")) if not di: log.error("Getting .discinfo for '%s' failed.", self.url) return 0 (self.name, self.version, self.arch) = di if self.name.startswith("Red Hat Enterprise Linux"): self.variant = self.name[24:].strip() self.id = "RHEL" self.prefix = "RedHat" elif self.name.startswith("Fedora"): self.variant = "" self.id = "FC" self.prefix = "Fedora" else: log.error("Unknown source '%s'.", self.name) return 0 self.release = "%s-%s" % (self.id, self.version) log.info1("Installation source: %s %s [%s]", self.name, self.version, self.arch) # load repos repos = [] self.yumconf = YumConf(self.version, self.arch, rpmdb=None, filenames=[], reposdirs=[]) if self.isRHEL() and self.cmpVersion("4.9") >= 0: # RHEL-5 key = None skip = False if ks.has_key("key"): key = ks["key"].keys()[0] if ks["key"][key].has_key("skip"): skip = True key = None inum = None if key and not skip and not beta_key_verify: if not instnum: log.warning("Unable to verify installation key, " "module instkey is missing.. using default" "installation.") else: try: inum = instnum.InstNum(key) except: log.error("Installation key '%s' is not valid.", key) return 0 if inum: if inum.get_product_string().lower() != \ self.variant.lower(): log.error( "Installation key for '%s' does not match " "'%s' media.", inum.get_product_string().lower(), self.variant.lower()) return 0 for name, path in inum.get_repos_dict().items(): if path == "VT" and \ not self.arch in [ "i386", "x86_64", "ia64" ]: continue repos.append(path) else: # BETA if self.variant == "Server": repos.append("Server") if key and key.find("C") >= 0: repos.append("Cluster") if key and key.find("S") >= 0: repos.append("ClusterStorage") elif self.variant == "Client": repos.append("Client") if key and key.find("W") >= 0: repos.append("Workstation") if self.arch in ["i386", "x86_64", "ia64"]: if key and key.find("V") >= 0: repos.append("VT") for repo in repos: repo_name = "%s-%s" % (self.release, repo) if repo in self.repos: log.error("Repository '%s' already defined.", repo_name) return 0 log.info1("Loading repo '%s'", repo_name) # create yumconf self.yumconf[repo_name] = {} self.yumconf[repo_name]["baseurl"] = [ "%s/%s" % (self.url, repo) ] if self.exclude: self.yumconf[repo_name]["exclude"] = self.exclude _repo = getRepoDB(rpmconfig, self.yumconf, reponame=repo_name) self.repos[repo_name] = _repo if not _repo.read(): log.error("Could not load repository '%s'.", repo_name) return 0 else: # RHEL <= 4 # FC repo = self.release self.yumconf[repo] = {} self.yumconf[repo]["baseurl"] = [self.url] if self.exclude: self.yumconf[repo]["exclude"] = self.exclude _repo = getRepoDB(rpmconfig, self.yumconf, reponame=repo) self.repos[repo] = _repo if not _repo.read(): log.error("Could not load repository '%s'.", repo) return 0 if not _repo.comps: # every source repo has to have a comps log.error("Missing comps file for '%s'.", repo) return 0 self.base_repo_names = self.repos.keys() if not ks.has_key("repo"): return 1 for repo in ks["repo"]: if repo in self.repos: log.error("Repository '%s' already defined.", repo) return 0 log.info1("Loading repository '%s'", repo) self.yumconf[repo] = {} url = ks["repo"][repo]["baseurl"] if url[:6] == "nfs://": d = "%s/%s" % (dir, repo) create_dir("", d) url = mount_nfs(url, d) self.yumconf[repo]["baseurl"] = [url] if ks["repo"][repo].has_key("exclude"): self.yumconf[repo]["exclude"] = ks["repo"][repo]["exclude"] if ks["repo"][repo].has_key("mirrorlist"): self.yumconf[repo]["mirrorlist"] = \ ks["repo"][repo]["mirrorlist"] _repo = getRepoDB(rpmconfig, self.yumconf, reponame=repo) self.repos[repo] = _repo if not _repo.read(): log.error("Could not load repository '%s'.", repo) return 0 return 1 def getStage2(self): if self.isRHEL() and self.cmpVersion("4.9") >= 0: return self.cache.cache("images/stage2.img") else: if self.cmpVersion("6") < 0: return self.cache.cache("%s/base/stage2.img" % self.prefix) else: return self.cache.cache("images/stage2.img") def getPackages(self, ks, languages, all_comps, has_raid, fstypes): groups = [] pkgs = [] everything = False if ks.has_key("packages") and \ ks["packages"].has_key("groups") and \ len(ks["packages"]["groups"]) > 0: groups = ks["packages"]["groups"] # add default group "base" and "core if it is not in groups and # nobase is not set if not ks.has_key("packages") or not ks["packages"].has_key("nobase"): if not "base" in groups: groups.append("base") if not "core" in groups: groups.append("core") if all_comps: repos = self.repos.keys() else: repos = self.base_repo_names if "everything" in groups: for repo in repos: for group in self.repos[repo].comps.getGroups(): if not group in groups: groups.append(group) groups.remove("everything") everything = True if ks.has_key("packages") and ks["packages"].has_key("add") and \ "*" in ks["packages"]["add"]: # add all packages for repo in self.repos.keys(): pkgs.extend(self.repos[repo].getNames()) else: # add default desktop if ks.has_key("xconfig"): if ks["xconfig"].has_key("startxonboot"): if not "base-x" in groups: log.info1("Adding group 'base-x'.") groups.append("base-x") desktop = "GNOME" if ks["xconfig"].has_key("defaultdesktop"): desktop = ks["xconfig"]["defaultdesktop"] desktop = "%s-desktop" % desktop.lower() if not desktop in groups: log.info1("Adding group '%s'.", desktop) groups.append(desktop) normalizeList(groups) # test if groups are available repo_groups = {} for group in groups: found = False for repo in repos: if not self.repos[repo].comps: continue _group = self.repos[repo].comps.getGroup(group) if not _group: continue found = True if not _group in repo_groups.keys() or \ not repo in repo_groups[_group]: repo_groups.setdefault(_group, []).append(repo) if not found: log.warning("Group '%s' does not exist.", group) del groups # add packages for groups for group in repo_groups: for repo in repo_groups[group]: comps = self.repos[repo].comps for pkg in comps.getPackageNames(group): if len(self.repos[repo].searchPkgs([pkg])) > 0: if not pkg in pkgs: pkgs.append(pkg) if everything: # add all packages in this group for (pkg, req) in \ comps.getConditionalPackageNames(group): if len(self.repos[repo].searchPkgs([pkg])) > 0: if not pkg in pkgs: pkgs.append(pkg) del repo_groups # add packages if ks.has_key("packages") and ks["packages"].has_key("add"): for name in ks["packages"]["add"]: if name == "*": continue found = False for repo in self.repos.keys(): _pkgs = self.repos[repo].searchPkgs([name]) if len(_pkgs) > 0: # silently add package if not name in pkgs: pkgs.append(name) found = True break if not found: log.warning("Package '%s' is not available.", pkg) # remove packages if ks.has_key("packages") and ks["packages"].has_key("drop"): for pkg in ks["packages"]["drop"]: if pkg in pkgs: log.info1("Removing package '%s'.", pkg) pkgs.remove(pkg) # add xorg driver package for past FC-5, RHEL-4 if ks.has_key("xconfig"): if (self.isRHEL() and self.cmpVersion("4.9") > 0) or \ (self.isFedora() and self.cmpVersion("4") > 0): if ks["xconfig"].has_key("driver"): self._addPkg("xorg-x11-drv-%s" % ks["xconfig"]["driver"], pkgs) else: if not "xorg-x11-drivers" in pkgs: self._addPkg("xorg-x11-drivers", pkgs) # add packages for needed filesystem types for fstype in fstypes: if fstype == "swap": continue self._addPkgByFilename("/sbin/mkfs.%s" % fstype, pkgs, "%s filesystem creation" % fstype) # add comps package if not "comps" in pkgs: try: self._addPkg("comps", pkgs) except: # ignore missing comps package pass # append mdadm if has_raid: self._addPkgByFilename("/sbin/mdadm", pkgs, "raid configuration") # append authconfig if ks.has_key("authconfig"): self._addPkgByFilename("/usr/sbin/authconfig", pkgs, "authentication configuration") # append iptables and config tool if ks.has_key("firewall") and \ not ks["firewall"].has_key("disabled"): self._addPkg("iptables", pkgs) # no firewall config tool in RHEL-3 if (self.isRHEL() and self.cmpVersion("4") >= 0) or \ (self.isFedora() and self.cmpVersion("3") >= 0): self._addPkgByFilename("/usr/sbin/lokkit", pkgs, "firewall configuration") # append lokkit if ks.has_key("selinux") and \ ((self.isRHEL() and self.cmpVersion("4") >= 0) or \ (self.isFedora() and self.cmpVersion("3") >= 0)): self._addPkgByFilename("/usr/sbin/lokkit", pkgs, "selinux configuration") # append kernel if not "kernel" in pkgs and not "kernel-smp" in pkgs: self._addPkg("kernel", pkgs) # append kernel-devel for FC-6 and RHEL-5 if "gcc" in pkgs and \ ((self.isRHEL() and self.cmpVersion("5") >= 0 and \ (self.getVariant() != "Client" or \ "%s-Workstation" % (self.release) in self.repos.keys())) or \ (self.isFedora() and self.cmpVersion("6") >= 0)): if "kernel" in pkgs: self._addPkg("kernel-devel", pkgs) elif "kernel-smp" in pkgs: self._addPkg("kernel-smp-devel", pkgs) # append firstboot if ks.has_key("firstboot") and \ not ks["firstboot"].has_key("disabled"): self._addPkg("firstboot", pkgs) # append dhclient if ks.has_key("bootloader"): self._addPkg("grub", pkgs) # if self.getArch() == "ia64": # self._addPkg("elilo", pkgs) # elif self.getArch in [ "s390", "s390x" ]: # self._addPkg("s390utils", pkgs) # elif self.getArch() in [ "ppc", "ppc64" ]: # self._addPkg("yaboot", pkgs) # else: # self._addPkg("grub", pkgs) # append grub if ks.has_key("network") and len(ks["network"]) > 0: for net in ks["network"]: if net["bootproto"] == "dhcp": self._addPkg("dhclient", pkgs) # languages (pre FC-6 and pre RHEL-5) if len(languages) > 0: for repo in repos: _repo = self.repos[repo] if not _repo.comps: continue for group in _repo.comps.grouphash.keys(): self._compsLangsupport(pkgs, _repo.comps, languages, group) normalizeList(pkgs) return pkgs def _addPkg(self, name, pkgs, description=""): for repo in self.repos: _pkgs = self.repos[repo].searchPkgs([name]) if len(_pkgs) > 0: if not name in pkgs: if description != "": log.info1("Adding package '%s' for %s.", name, description) else: log.info1("Adding package '%s'.", name) pkgs.append(name) return if description != "": raise ValueError, "Could not find package '%s'" % (name) + \ "needed for %s." % (description) else: raise ValueError, "Could not find package '%s'" % (name) def _addPkgByFilename(self, name, pkgs, description=""): for repo in self.repos: s = self.repos[repo].searchFilenames(name) if len(s) < 1: # import file list if not already imported and search again if not self.repos[repo]._matchesFile(name) and \ not self.repos[repo].isFilelistImported(): self.repos[repo].importFilelist() s = self.repos[repo].searchFilenames(name) if len(s) < 1: continue for pkg in s: if pkg["name"] in pkgs: # package is already in list return pkg = s[0] # take first package, which provides ... if description != "": log.info1("Adding package '%s' for %s.", pkg["name"], description) else: log.info1("Adding package '%s'.", pkg["name"]) pkgs.append(pkg["name"]) return if description != "": raise ValueError, "Could not find package providing '%s'" % \ (name) + "needed for %s." % (description) else: raise ValueError, "Could not find package providing '%s'" % (name) def _compsLangsupport(self, pkgs, comps, languages, group): if not comps.grouphash.has_key(group): return if not comps.grouphash[group].has_key("langonly") or \ not comps.grouphash[group]["langonly"] in languages: return # add grouplist if comps.grouphash[group].has_key("grouplist"): for _group in comps.grouphash[group]["grouplist"]["groupreqs"]: self._compsLangsupport(pkgs, comps, languages, _group) for _group in comps.grouphash[group]["grouplist"]["metapkgs"]: self._compsLangsupport(pkgs, comps, languages, _group) for name in comps.getPackageNames(group): self._addPkg(name, pkgs, "langsupport") # old style conditional optional_list = comps.getOptionalPackageNames(group) for (name, requires) in optional_list: if name in pkgs: continue for req in requires: if req in pkgs: log.info1("Adding package '%s' for langsupport.", name) pkgs.append(name) break # new style conditional conditional_list = comps.getConditionalPackageNames(group) for (name, requires) in conditional_list: if name in pkgs: continue for req in requires: if req in pkgs: log.info1("Adding package '%s' for langsupport.", name) pkgs.append(name) break def getYumConf(self): ret = "" for repo in self.repos: ret += "[%s]\n" % repo ret += "name=%s\n" % repo _repo = self.repos[repo] baseurls = _repo.nc.baseurls[_repo.reponame] if len(baseurls) > 0: ret += "baseurl=%s\n" % " ".join(baseurls) if _repo.excludes: ret += "exclude=%s\n" % " ".join(_repo.excludes) if _repo.mirrorlist: ret += "mirrorlist=%s\n" % " ".join(_repo.mirrorlist) ret += "\n" return ret def close(self): for repo in self.repos: self.repos[repo].close() self.repos.clear() def cleanup(self): self.close() for mount in self.mounts: self.mounts[mount].umount() def isRHEL(self): return self.id == "RHEL" def isFedora(self): return self.id == "FC" def getName(self): return self.name def getVersion(self): return self.version def getArch(self): return self.arch def cmpVersion(self, other): if self.getVersion() == "development": return 1 return stringCompare(self.getVersion(), other) def getRelease(self): return self.release def getVariant(self): return self.variant
class Source: """ Load Source repo and extra repos according to kickstart configuration. """ def __init__(self): self.repos = { } self.base_repo_names = [ ] self.mounts = { } def load(self, ks, dir, beta_key_verify=False): self.dir = dir self.exclude = None # mount source to dir if ks.has_key("cdrom"): self.url = mount_cdrom(dir) if ks["cdrom"].has_key("exclude"): self.exclude = ks["cdrom"]["exclude"] elif ks.has_key("nfs"): opts = None if ks["nfs"].has_key("opts"): opts = ks["nfs"]["opts"] self.url = mount_nfs("nfs://%s:%s" % \ (ks["nfs"]["server"], ks["nfs"]["dir"]), dir, options=opts) if ks["nfs"].has_key("exclude"): self.exclude = ks["nfs"]["exclude"] else: self.url = ks["url"]["url"] if ks["url"].has_key("exclude"): self.exclude = ks["url"]["exclude"] # create network cache self.cache = NetworkCache([ self.url ], cachedir=rpmconfig.cachedir) # get source information via .discinfo file if not self.cache.cache(".discinfo"): log.error("No .discinfo for '%s'", self.url) return 0 di = get_discinfo(self.cache.cache(".discinfo")) if not di: log.error("Getting .discinfo for '%s' failed.", self.url) return 0 (self.name, self.version, self.arch) = di if self.name.startswith("Red Hat Enterprise Linux"): self.variant = self.name[24:].strip() self.id = "RHEL" self.prefix = "RedHat" elif self.name.startswith("Fedora"): self.variant = "" self.id = "FC" self.prefix = "Fedora" else: log.error("Unknown source '%s'.", self.name) return 0 self.release = "%s-%s" % (self.id, self.version) log.info1("Installation source: %s %s [%s]", self.name, self.version, self.arch) # load repos repos = [ ] self.yumconf = YumConf(self.version, self.arch, rpmdb=None, filenames=[ ], reposdirs=[ ]) if self.isRHEL() and self.cmpVersion("4.9") >= 0: # RHEL-5 key = None skip = False if ks.has_key("key"): key = ks["key"].keys()[0] if ks["key"][key].has_key("skip"): skip = True key = None inum = None if key and not skip and not beta_key_verify: if not instnum: log.warning("Unable to verify installation key, " "module instkey is missing.. using default" "installation.") else: try: inum = instnum.InstNum(key) except: log.error("Installation key '%s' is not valid.", key) return 0 if inum: if inum.get_product_string().lower() != \ self.variant.lower(): log.error("Installation key for '%s' does not match " "'%s' media.", inum.get_product_string().lower(), self.variant.lower()) return 0 for name, path in inum.get_repos_dict().items(): if path == "VT" and \ not self.arch in [ "i386", "x86_64", "ia64" ]: continue repos.append(path) else: # BETA if self.variant == "Server": repos.append("Server") if key and key.find("C") >= 0: repos.append("Cluster") if key and key.find("S") >= 0: repos.append("ClusterStorage") elif self.variant == "Client": repos.append("Client") if key and key.find("W") >= 0: repos.append("Workstation") if self.arch in [ "i386", "x86_64", "ia64" ]: if key and key.find("V") >= 0: repos.append("VT") for repo in repos: repo_name = "%s-%s" % (self.release, repo) if repo in self.repos: log.error("Repository '%s' already defined.", repo_name) return 0 log.info1("Loading repo '%s'", repo_name) # create yumconf self.yumconf[repo_name] = { } self.yumconf[repo_name]["baseurl"] = [ "%s/%s" % (self.url, repo) ] if self.exclude: self.yumconf[repo_name]["exclude"] = self.exclude _repo = getRepoDB(rpmconfig, self.yumconf, reponame=repo_name) self.repos[repo_name] = _repo if not _repo.read(): log.error("Could not load repository '%s'.", repo_name) return 0 else: # RHEL <= 4 # FC repo = self.release self.yumconf[repo] = { } self.yumconf[repo]["baseurl"] = [ self.url ] if self.exclude: self.yumconf[repo]["exclude"] = self.exclude _repo = getRepoDB(rpmconfig, self.yumconf, reponame=repo) self.repos[repo] = _repo if not _repo.read(): log.error("Could not load repository '%s'.", repo) return 0 if not _repo.comps: # every source repo has to have a comps log.error("Missing comps file for '%s'.", repo) return 0 self.base_repo_names = self.repos.keys() if not ks.has_key("repo"): return 1 for repo in ks["repo"]: if repo in self.repos: log.error("Repository '%s' already defined.", repo) return 0 log.info1("Loading repository '%s'", repo) self.yumconf[repo] = { } url = ks["repo"][repo]["baseurl"] if url[:6] == "nfs://": d = "%s/%s" % (dir, repo) create_dir("", d) url = mount_nfs(url, d) self.yumconf[repo]["baseurl"] = [ url ] if ks["repo"][repo].has_key("exclude"): self.yumconf[repo]["exclude"] = ks["repo"][repo]["exclude"] if ks["repo"][repo].has_key("mirrorlist"): self.yumconf[repo]["mirrorlist"] = \ ks["repo"][repo]["mirrorlist"] _repo = getRepoDB(rpmconfig, self.yumconf, reponame=repo) self.repos[repo] = _repo if not _repo.read(): log.error("Could not load repository '%s'.", repo) return 0 return 1 def getStage2(self): if self.isRHEL() and self.cmpVersion("4.9") >= 0: return self.cache.cache("images/stage2.img") else: if self.cmpVersion("6") < 0: return self.cache.cache("%s/base/stage2.img" % self.prefix) else: return self.cache.cache("images/stage2.img") def getPackages(self, ks, languages, all_comps, has_raid, fstypes): groups = [ ] pkgs = [ ] everything = False if ks.has_key("packages") and \ ks["packages"].has_key("groups") and \ len(ks["packages"]["groups"]) > 0: groups = ks["packages"]["groups"] # add default group "base" and "core if it is not in groups and # nobase is not set if not ks.has_key("packages") or not ks["packages"].has_key("nobase"): if not "base" in groups: groups.append("base") if not "core" in groups: groups.append("core") if all_comps: repos = self.repos.keys() else: repos = self.base_repo_names if "everything" in groups: for repo in repos: for group in self.repos[repo].comps.getGroups(): if not group in groups: groups.append(group) groups.remove("everything") everything = True if ks.has_key("packages") and ks["packages"].has_key("add") and \ "*" in ks["packages"]["add"]: # add all packages for repo in self.repos.keys(): pkgs.extend(self.repos[repo].getNames()) else: # add default desktop if ks.has_key("xconfig"): if ks["xconfig"].has_key("startxonboot"): if not "base-x" in groups: log.info1("Adding group 'base-x'.") groups.append("base-x") desktop = "GNOME" if ks["xconfig"].has_key("defaultdesktop"): desktop = ks["xconfig"]["defaultdesktop"] desktop = "%s-desktop" % desktop.lower() if not desktop in groups: log.info1("Adding group '%s'.", desktop) groups.append(desktop) normalizeList(groups) # test if groups are available repo_groups = { } for group in groups: found = False for repo in repos: if not self.repos[repo].comps: continue _group = self.repos[repo].comps.getGroup(group) if not _group: continue found = True if not _group in repo_groups.keys() or \ not repo in repo_groups[_group]: repo_groups.setdefault(_group, [ ]).append(repo) if not found: log.warning("Group '%s' does not exist.", group) del groups # add packages for groups for group in repo_groups: for repo in repo_groups[group]: comps = self.repos[repo].comps for pkg in comps.getPackageNames(group): if len(self.repos[repo].searchPkgs([pkg])) > 0: if not pkg in pkgs: pkgs.append(pkg) if everything: # add all packages in this group for (pkg, req) in \ comps.getConditionalPackageNames(group): if len(self.repos[repo].searchPkgs([pkg])) > 0: if not pkg in pkgs: pkgs.append(pkg) del repo_groups # add packages if ks.has_key("packages") and ks["packages"].has_key("add"): for name in ks["packages"]["add"]: if name == "*": continue found = False for repo in self.repos.keys(): _pkgs = self.repos[repo].searchPkgs([name]) if len(_pkgs) > 0: # silently add package if not name in pkgs: pkgs.append(name) found = True break if not found: log.warning("Package '%s' is not available.", pkg) # remove packages if ks.has_key("packages") and ks["packages"].has_key("drop"): for pkg in ks["packages"]["drop"]: if pkg in pkgs: log.info1("Removing package '%s'.", pkg) pkgs.remove(pkg) # add xorg driver package for past FC-5, RHEL-4 if ks.has_key("xconfig"): if (self.isRHEL() and self.cmpVersion("4.9") > 0) or \ (self.isFedora() and self.cmpVersion("4") > 0): if ks["xconfig"].has_key("driver"): self._addPkg("xorg-x11-drv-%s" % ks["xconfig"]["driver"], pkgs) else: if not "xorg-x11-drivers" in pkgs: self._addPkg("xorg-x11-drivers", pkgs) # add packages for needed filesystem types for fstype in fstypes: if fstype == "swap": continue self._addPkgByFilename("/sbin/mkfs.%s" % fstype, pkgs, "%s filesystem creation" % fstype) # add comps package if not "comps" in pkgs: try: self._addPkg("comps", pkgs) except: # ignore missing comps package pass # append mdadm if has_raid: self._addPkgByFilename("/sbin/mdadm", pkgs, "raid configuration") # append authconfig if ks.has_key("authconfig"): self._addPkgByFilename("/usr/sbin/authconfig", pkgs, "authentication configuration") # append iptables and config tool if ks.has_key("firewall") and \ not ks["firewall"].has_key("disabled"): self._addPkg("iptables", pkgs) # no firewall config tool in RHEL-3 if (self.isRHEL() and self.cmpVersion("4") >= 0) or \ (self.isFedora() and self.cmpVersion("3") >= 0): self._addPkgByFilename("/usr/sbin/lokkit", pkgs, "firewall configuration") # append lokkit if ks.has_key("selinux") and \ ((self.isRHEL() and self.cmpVersion("4") >= 0) or \ (self.isFedora() and self.cmpVersion("3") >= 0)): self._addPkgByFilename("/usr/sbin/lokkit", pkgs, "selinux configuration") # append kernel if not "kernel" in pkgs and not "kernel-smp" in pkgs: self._addPkg("kernel", pkgs) # append kernel-devel for FC-6 and RHEL-5 if "gcc" in pkgs and \ ((self.isRHEL() and self.cmpVersion("5") >= 0 and \ (self.getVariant() != "Client" or \ "%s-Workstation" % (self.release) in self.repos.keys())) or \ (self.isFedora() and self.cmpVersion("6") >= 0)): if "kernel" in pkgs: self._addPkg("kernel-devel", pkgs) elif "kernel-smp" in pkgs: self._addPkg("kernel-smp-devel", pkgs) # append firstboot if ks.has_key("firstboot") and \ not ks["firstboot"].has_key("disabled"): self._addPkg("firstboot", pkgs) # append dhclient if ks.has_key("bootloader"): self._addPkg("grub", pkgs) # if self.getArch() == "ia64": # self._addPkg("elilo", pkgs) # elif self.getArch in [ "s390", "s390x" ]: # self._addPkg("s390utils", pkgs) # elif self.getArch() in [ "ppc", "ppc64" ]: # self._addPkg("yaboot", pkgs) # else: # self._addPkg("grub", pkgs) # append grub if ks.has_key("network") and len(ks["network"]) > 0: for net in ks["network"]: if net["bootproto"] == "dhcp": self._addPkg("dhclient", pkgs) # languages (pre FC-6 and pre RHEL-5) if len(languages) > 0: for repo in repos: _repo = self.repos[repo] if not _repo.comps: continue for group in _repo.comps.grouphash.keys(): self._compsLangsupport(pkgs, _repo.comps, languages, group) normalizeList(pkgs) return pkgs def _addPkg(self, name, pkgs, description=""): for repo in self.repos: _pkgs = self.repos[repo].searchPkgs([name]) if len(_pkgs) > 0: if not name in pkgs: if description != "": log.info1("Adding package '%s' for %s.", name, description) else: log.info1("Adding package '%s'.", name) pkgs.append(name) return if description != "": raise ValueError, "Could not find package '%s'" % (name) + \ "needed for %s." % (description) else: raise ValueError, "Could not find package '%s'" % (name) def _addPkgByFilename(self, name, pkgs, description=""): for repo in self.repos: s = self.repos[repo].searchFilenames(name) if len(s) < 1: # import file list if not already imported and search again if not self.repos[repo]._matchesFile(name) and \ not self.repos[repo].isFilelistImported(): self.repos[repo].importFilelist() s = self.repos[repo].searchFilenames(name) if len(s) < 1: continue for pkg in s: if pkg["name"] in pkgs: # package is already in list return pkg = s[0] # take first package, which provides ... if description != "": log.info1("Adding package '%s' for %s.", pkg["name"], description) else: log.info1("Adding package '%s'.", pkg["name"]) pkgs.append(pkg["name"]) return if description != "": raise ValueError, "Could not find package providing '%s'" % \ (name) + "needed for %s." % (description) else: raise ValueError, "Could not find package providing '%s'" % (name) def _compsLangsupport(self, pkgs, comps, languages, group): if not comps.grouphash.has_key(group): return if not comps.grouphash[group].has_key("langonly") or \ not comps.grouphash[group]["langonly"] in languages: return # add grouplist if comps.grouphash[group].has_key("grouplist"): for _group in comps.grouphash[group]["grouplist"]["groupreqs"]: self._compsLangsupport(pkgs, comps, languages, _group) for _group in comps.grouphash[group]["grouplist"]["metapkgs"]: self._compsLangsupport(pkgs, comps, languages, _group) for name in comps.getPackageNames(group): self._addPkg(name, pkgs, "langsupport") # old style conditional optional_list = comps.getOptionalPackageNames(group) for (name, requires) in optional_list: if name in pkgs: continue for req in requires: if req in pkgs: log.info1("Adding package '%s' for langsupport.", name) pkgs.append(name) break # new style conditional conditional_list = comps.getConditionalPackageNames(group) for (name, requires) in conditional_list: if name in pkgs: continue for req in requires: if req in pkgs: log.info1("Adding package '%s' for langsupport.", name) pkgs.append(name) break def getYumConf(self): ret = "" for repo in self.repos: ret += "[%s]\n" % repo ret += "name=%s\n" % repo _repo = self.repos[repo] baseurls = _repo.nc.baseurls[_repo.reponame] if len(baseurls) > 0: ret += "baseurl=%s\n" % " ".join(baseurls) if _repo.excludes: ret += "exclude=%s\n" % " ".join(_repo.excludes) if _repo.mirrorlist: ret += "mirrorlist=%s\n" % " ".join(_repo.mirrorlist) ret += "\n" return ret def close(self): for repo in self.repos: self.repos[repo].close() self.repos.clear() def cleanup(self): self.close() for mount in self.mounts: self.mounts[mount].umount() def isRHEL(self): return self.id == "RHEL" def isFedora(self): return self.id == "FC" def getName(self): return self.name def getVersion(self): return self.version def getArch(self): return self.arch def cmpVersion(self, other): if self.getVersion() == "development": return 1 return stringCompare(self.getVersion(), other) def getRelease(self): return self.release def getVariant(self): return self.variant
class RpmRepoDB(memorydb.RpmMemoryDB): """A (mostly) read-only RPM database storage in repodata XML. This is not a full implementation of Database: notably the file database is not populated at all.""" # A mapping between strings and RPMSENSE_* comparison flags flagmap = { 0: None, None: 0, "EQ": RPMSENSE_EQUAL, "LT": RPMSENSE_LESS, "GT": RPMSENSE_GREATER, "LE": RPMSENSE_EQUAL | RPMSENSE_LESS, "GE": RPMSENSE_EQUAL | RPMSENSE_GREATER, RPMSENSE_EQUAL: "EQ", RPMSENSE_LESS: "LT", RPMSENSE_GREATER: "GT", RPMSENSE_EQUAL | RPMSENSE_LESS: "LE", RPMSENSE_EQUAL | RPMSENSE_GREATER: "GE" } def __init__(self, config, source, buildroot='', reponame="default", nc=None): """Exclude packages matching whitespace-separated excludes. Use reponame for cache subdirectory name and pkg["yumreponame"]. Load PGP keys from URLs in key_urls.""" memorydb.RpmMemoryDB.__init__(self, config, source, buildroot) self.reponame = reponame self.excludes = self.config.excludes[:] self.mirrorlist = None self.baseurls = None self.yumconf = None self.key_urls = [] if nc: self.nc = nc else: self.nc = NetworkCache([], self.config.cachedir, self.reponame) if isinstance(source, types.DictType): found_urls = False self.yumconf = source if self.yumconf.has_key("main"): sec = self.yumconf["main"] if sec.has_key("exclude"): self.excludes.extend(sec["exclude"]) sec = self.yumconf[self.reponame] if sec.has_key("exclude"): self.excludes.extend(sec["exclude"]) if sec.has_key("gpgkey"): self.key_urls = sec["gpgkey"] if sec.has_key("baseurl"): self.nc.addCache(sec["baseurl"], self.reponame) found_urls = True if sec.has_key("mirrorlist"): self.mirrorlist = sec["mirrorlist"] found_urls = True if not found_urls: raise ValueError, "yum.conf is missing mirrorlist or baseurl parameter" else: self.baseurls = source self.nc.addCache(self.baseurls, self.reponame) self.repomd = None self.filelist_imported = 0 # Files included in primary.xml self._filerc = re.compile('^(.*bin/.*|/etc/.*|/usr/lib/sendmail)$') self._dirrc = re.compile('^(.*bin/.*|/etc/.*)$') self.comps = None def readMirrorList(self): if not self.is_read and self.mirrorlist and self.yumconf: fname = self.nc.cache(self.mirrorlist, 1) if fname: lines = open(fname).readlines() os.unlink(fname) else: lines = [] for l in lines: l = l.strip() l = l.replace("$ARCH", "$basearch") l = self.yumconf.replaceVars(l) if l and l[0] != "#": self.nc.addCache([ l, ]) def getExcludes(self): return self.excludes def getMirrorList(self): return self.mirrorlist def isIdentitySave(self): """return if package objects that are added are in the db afterwards (.__contains__() returns True and the object are return from searches) """ return False def readRepoMD(self): # First we try and read the repomd file as a starting point. filename = self.nc.cache("repodata/repomd.xml", 1) if not filename: log.error("Couldn't open repomd.xml") return 0 try: fd = open(filename) ip = iterparse(fd, events=("start", "end")) ip = iter(ip) except IOError: log.error("Couldn't parse repomd.xml") return 0 # Create our network cache object self.repomd = self._parse(ip) return 1 def readComps(self): # Try to read a comps.xml file if there is any before we parse the # primary.xml if self.repomd.has_key("group"): if not self.repomd["group"].has_key("location"): log.error( "Couldn't find proper location for comps.xml in repomd") return 0 comps = self.repomd["group"]["location"] (csum, destfile) = self.nc.checksum(comps, "sha") if self.repomd["group"].has_key("checksum") and \ csum == self.repomd["group"]["checksum"]: filename = destfile else: filename = self.nc.cache(comps, 1) if not filename: return 0 try: self.comps = RpmCompsXML(self.config, filename) self.comps.read() except IOError: return 0 return 1 def readPrimary(self): # If we have either a local cache of the primary.xml.gz file or if # it is already local (nfs or local file system) we calculate it's # checksum and compare it with the one from repomd. If they are # the same we don't need to cache it again and can directly use it. if self.repomd.has_key("primary"): if not self.repomd["primary"].has_key("location"): print "Error primary has no location" return 0 primary = self.repomd["primary"]["location"] # if self.repomd["primary"].has_key("checksum"): # (csum, destfile) = self.nc.checksum(primary, "sha") # csum == self.repomd["primary"]["checksum"]: # filename = destfile # else: filename = self.nc.cache(primary, 1) if not filename: print "Error can't find file for primary: " + primary return 0 try: fd = PyGZIP(filename) ip = iterparse(fd, events=("start", "end")) ip = iter(ip) except IOError: log.error("Couldn't parse primary.xml") print "Error parsing primary.xml" return 0 self._parse(ip) return 1 def readPGPKeys(self): for url in self.key_urls: filename = self.nc.cache(url, 1) try: f = file(filename) key_data = f.read() f.close() except Exception, e: log.error("Error reading GPG key %s: %s", filename, e) continue try: key_data = openpgp.isolateASCIIArmor(key_data) keys = openpgp.parsePGPKeys(key_data) except Exception, e: log.error("Invalid GPG key %s: %s", url, e) continue for k in keys: self.keyring.addKey(k)