def _parse(self, ip): """Parse <package> tags.""" for event, elem in ip: tag = elem.tag if event != "start": continue if not tag.endswith("}package") and \ not tag.endswith("}repomd"): continue if tag.endswith("}repomd"): return self.__parseRepomd(ip) props = elem.attrib if props.get("type") == "rpm": try: pkg = self.__parsePackage(ip) except ValueError, e: log.warning("%s: %s", pkg.getNEVRA(), e) continue pkg.yumrepo = self if self.comps != None: if self.comps.hasType(pkg["name"], "mandatory"): pkg.compstype = "mandatory" elif self.comps.hasType(pkg["name"], "default"): pkg.compstype = "default" elif self.comps.hasType(pkg["name"], "optional"): pkg.compstype = "optional" self.addPkg(pkg) elif props.has_key("name"): arch = props.get("arch") if arch == None: log.warning("%s: missing arch= in <package>", pkg.getNEVRA()) continue self.__parseFilelist(ip, props["name"], arch)
def _parse(self, ip): """Parse <package> tags.""" for event, elem in ip: tag = elem.tag if event != "start": continue if not tag.endswith("}package") and \ not tag.endswith("}repomd"): continue if tag.endswith("}repomd"): return self.__parseRepomd(ip) props = elem.attrib if props.get("type") == "rpm": try: pkg = self.__parsePackage(ip) except ValueError, e: log.warning("%s: %s", ip, e) continue pkg.yumrepo = self if self.comps != None: if self.comps.hasType(pkg["name"], "mandatory"): pkg.compstype = "mandatory" elif self.comps.hasType(pkg["name"], "default"): pkg.compstype = "default" elif self.comps.hasType(pkg["name"], "optional"): pkg.compstype = "optional" self.addPkg(pkg) elif props.has_key("name"): arch = props.get("arch") if arch == None: log.warning("%s: missing arch= in <package>", pkg.getNEVRA()) continue self.__parseFilelist(ip, props["name"], arch)
def filterArchCompat(list, arch): """Modify RpmPackage list list to contain only packages that can be installed on arch arch. Warn using config.rpmconfig about dropped packages.""" i = 0 while i < len(list): if archCompat(list[i]["arch"], arch): i += 1 else: log.warning("%s: Architecture not compatible with %s", list[i].source, arch) list.pop(i)
def __parseRepomd(self, ip): """Parse repomd.xml for SHA1 checks of the files. Returns a hash of the form: name -> {location, checksum, timestamp, open-checksum}""" rethash = {} # Make local variables for heavy used functions to speed up this loop tmphash = {} fname = None for event, elem in ip: isend = (event == "end") props = elem.attrib tag = elem.tag if not isend and tag.endswith("}data"): fname = props.get("type") if not fname: break tmphash = {} rethash[fname] = tmphash if not isend: continue if tag.endswith("}repomd"): break elif tag.endswith("}location"): loc = props.get("href") if loc: tmphash["location"] = loc elif tag.endswith("}checksum"): type = props.get("type") if type != "sha" and type != "sha256": log.warning( "Unsupported checksum type %s in repomd.xml " "for file %s", type, fname) continue tmphash["checksum"] = elem.text tmphash["checksum.type"] = type elif tag.endswith("}timestamp"): tmphash["timestamp"] = elem.text elif tag.endswith("}open-checksum"): type = props.get("type") if type != "sha" and type != "sha256": log.warning( "Unsupported open-checksum type %s in " "repomd.xml for file %s", type, fname) continue tmphash["open-checksum"] = elem.text tmphash["open-checksum.type"] = type return rethash
def _isExcluded(self, pkg): """Return True if RpmPackage pkg is excluded by configuration.""" if pkg["arch"] == "src": return 1 if not self.config.ignorearch and \ (not functions.archCompat(pkg["arch"], self.config.machine) or \ (self.config.archlist != None and not pkg["arch"] in self.config.archlist)) and \ not pkg.isSourceRPM(): log.warning("%s: Package excluded because of arch " "incompatibility", pkg.getNEVRA()) return 1 index = lists.NevraList() index.addPkg(pkg) result = index.search(self.excludes) return bool(result)
def updateDigestFromRegion(self, digest, region, header_pos): if region is None or len(region) != 16: # What was the digest computed from? raise ValueError, "No region" (tag, type_, offset, count) = unpack("!2IiI", region) # FIXME: other regions than "immutable"? if (tag != 63 or type_ != RPM_BIN or -offset <= 0 or -offset % 16 != 0 or count != 16): raise ValueError, "Invalid region" regionIndexEntries = -offset / 16 if header_pos[0] is None: raise NotImplementedError fd = self.__getFdForRange(*header_pos) data = fd.read(16) if len(data) != 16: raise ValueError, "Unexpected EOF in header" (totalIndexEntries, totalDataSize) = unpack("!8x2I", data) data = fd.read(16 * totalIndexEntries) if len(data) != 16 * totalIndexEntries: raise ValueError, "Unexpected EOF in header" unsignedTags = [] for i in xrange(totalIndexEntries): (tag, type_, offset, count) = \ unpack("!4I", data[i * 16 : (i + 1) * 16]) # FIXME: other regions than "immutable"? if tag == 63: break unsignedTags.append(tag) else: raise ValueError, "%s: immutable tag disappeared" % self.source if (type_ != RPM_BIN or count != 16 or i + regionIndexEntries > totalIndexEntries): raise ValueError, "Invalid region tag" digest.update(pack("!2I", regionIndexEntries, offset + 16)) digest.update(data[i * 16 : (i + regionIndexEntries) * 16]) for i in xrange(i + regionIndexEntries, totalIndexEntries): (tag,) = unpack("!I", data[i * 16 : i * 16 + 4]) unsignedTags.append(tag) if unsignedTags: # FIXME: only once per package log.warning("%s: Unsigned tags %s", self.source, [rpmtagname[i] for i in unsignedTags]) # In practice region data starts at offset 0, but the original design # was proposing concatenated regions etc; where would the data region # start in that case? Lowest offset in region perhaps? functions.updateDigestFromFile(digest, fd, offset + 16)
def updateDigestFromRegion(self, digest, region, header_pos): if region is None or len(region) != 16: # What was the digest computed from? raise ValueError, "No region" (tag, type_, offset, count) = unpack("!2IiI", region) # FIXME: other regions than "immutable"? if (tag != 63 or type_ != RPM_BIN or -offset <= 0 or -offset % 16 != 0 or count != 16): raise ValueError, "Invalid region" regionIndexEntries = -offset / 16 if header_pos[0] is None: raise NotImplementedError fd = self.__getFdForRange(*header_pos) data = fd.read(16) if len(data) != 16: raise ValueError, "Unexpected EOF in header" (totalIndexEntries, totalDataSize) = unpack("!8x2I", data) data = fd.read(16 * totalIndexEntries) if len(data) != 16 * totalIndexEntries: raise ValueError, "Unexpected EOF in header" unsignedTags = [] for i in xrange(totalIndexEntries): (tag, type_, offset, count) = \ unpack("!4I", data[i * 16 : (i + 1) * 16]) # FIXME: other regions than "immutable"? if tag == 63: break unsignedTags.append(tag) else: raise ValueError, "%s: immutable tag disappeared" % self.source if (type_ != RPM_BIN or count != 16 or i + regionIndexEntries > totalIndexEntries): raise ValueError, "Invalid region tag" digest.update(pack("!2I", regionIndexEntries, offset + 16)) digest.update(data[i * 16:(i + regionIndexEntries) * 16]) for i in xrange(i + regionIndexEntries, totalIndexEntries): (tag, ) = unpack("!I", data[i * 16:i * 16 + 4]) unsignedTags.append(tag) if unsignedTags: # FIXME: only once per package log.warning("%s: Unsigned tags %s", self.source, [rpmtagname[i] for i in unsignedTags]) # In practice region data starts at offset 0, but the original design # was proposing concatenated regions etc; where would the data region # start in that case? Lowest offset in region perhaps? functions.updateDigestFromFile(digest, fd, offset + 16)
def __parseRepomd(self, ip): """Parse repomd.xml for SHA1 checks of the files. Returns a hash of the form: name -> {location, checksum, timestamp, open-checksum}""" rethash = {} # Make local variables for heavy used functions to speed up this loop tmphash = {} fname = None for event, elem in ip: isend = (event == "end") props = elem.attrib tag = elem.tag if not isend and tag.endswith("}data"): fname = props.get("type") if not fname: break tmphash = {} rethash[fname] = tmphash if not isend: continue if tag.endswith("}repomd"): break elif tag.endswith("}location"): loc = props.get("href") if loc: tmphash["location"] = loc elif tag.endswith("}checksum"): type = props.get("type") if type != "sha" and type != "sha256": log.warning("Unsupported checksum type %s in repomd.xml " "for file %s", type, fname) continue tmphash["checksum"] = elem.text tmphash["checksum.type"] = type elif tag.endswith("}timestamp"): tmphash["timestamp"] = elem.text elif tag.endswith("}open-checksum"): type = props.get("type") if type != "sha" and type != "sha256": log.warning("Unsupported open-checksum type %s in " "repomd.xml for file %s", type, fname) continue tmphash["open-checksum"] = elem.text tmphash["open-checksum.type"] = type return rethash
def __parse(self, ip): """Parse node and its siblings under the root element. Return 1 on success, 0 on failure. Handle <group>, <grouphierarchy>, warn about other tags.""" for event, elem in ip: tag = elem.tag if tag == "comps": continue elif tag == "group" or tag == "category": self.__parseGroup(ip) elif tag == "grouphierarchy": ret = self.__parseGroupHierarchy(ip) else: log.warning("Unknown entry in comps.xml: %s", tag) return 0 return 1
def _isExcluded(self, pkg): """Return True if RpmPackage pkg is excluded by configuration.""" if pkg["arch"] == "src": return 1 if not self.config.ignorearch and \ (not functions.archCompat(pkg["arch"], self.config.machine) or \ (self.config.archlist != None and not pkg["arch"] in self.config.archlist)) and \ not pkg.isSourceRPM(): log.warning( "%s: Package excluded because of arch " "incompatibility", pkg.getNEVRA()) return 1 index = lists.NevraList() index.addPkg(pkg) result = index.search(self.excludes) return bool(result)
def create(self, filename): """Create an initial database""" # If it exists, remove it as we were asked to create a new one if os.path.exists(filename): try: os.unlink(filename) except OSError: pass # Try to create the databse in filename, or use in memory when # this fails try: f = open(filename, 'w') db = sqlite3.connect(filename) except IOError: log.warning("Could not create sqlite cache file, using in memory " "cache instead") db = sqlite3.connect(":memory:") db.row_factory = sqlite3.Row db.text_factory = str return db
def __parse(self, ip): """Parse node and its siblings under the root element. Return 1 on success, 0 on failure. Handle <group>, <grouphierarchy>, warn about other tags.""" for event, elem in ip: tag = elem.tag if tag == "comps": continue elif tag == "group" or tag == "category": self.__parseGroup(ip) elif tag == "grouphierarchy": ret = self.__parseGroupHierarchy(ip) elif tag == "blacklist" or tag == "whiteout": for ev2, el2 in ip: if ev2 == "end" and (el2.tag == "blacklist" or el2.tag == "whiteout"): break else: log.warning("Unknown entry in comps.xml: %s", tag) return 0 return 1
#odoc = libxml2.newDoc("1.0") #oroot = odoc.newChild(None, "filelists", None) log.info1("Pass 2: Writing repodata information.") pfd.write('<?xml version="1.0" encoding="UTF-8"?>\n') pfd.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%d">\n' % len(self.getPkgs())) ffd.write('<?xml version="1.0" encoding="UTF-8"?>\n') ffd.write('<filelists xmlns:rpm="http://linux.duke.edu/filelists" packages="%d">\n' % len(self.getPkgs())) for pkg in self.getPkgs(): log.info2("Processing complete data of package %s.", pkg.getNEVRA()) pkg.header_read = 0 try: pkg.open() pkg.read() except (IOError, ValueError), e: log.warning("%s: %s", pkg.getNEVRA(), e) continue # If it is a source rpm change the arch to "src". Only valid # for createRepo, never do this anywhere else. ;) if pkg.isSourceRPM(): pkg["arch"] = "src" try: checksum = self.__getChecksum(pkg) except (IOError, NotImplementedError), e: log.warning("%s: %s", pkg.getNEVRA(), e) continue pkg["yumchecksum"] = checksum self.__writePrimary(pfd, proot, pkg) self.__writeFilelists(ffd, froot, pkg) # self.__writeOther(ofd, oroot, pkg) try:
pfd.write( '<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%d">\n' % len(self.getPkgs())) ffd.write('<?xml version="1.0" encoding="UTF-8"?>\n') ffd.write( '<filelists xmlns:rpm="http://linux.duke.edu/filelists" packages="%d">\n' % len(self.getPkgs())) for pkg in self.getPkgs(): log.info2("Processing complete data of package %s.", pkg.getNEVRA()) pkg.header_read = 0 try: pkg.open() pkg.read() except (IOError, ValueError), e: log.warning("%s: %s", pkg.getNEVRA(), e) continue # If it is a source rpm change the arch to "src". Only valid # for createRepo, never do this anywhere else. ;) if pkg.isSourceRPM(): pkg["arch"] = "src" try: checksum = self.__getChecksum(pkg) except (IOError, NotImplementedError), e: log.warning("%s: %s", pkg.getNEVRA(), e) continue pkg["yumchecksum"] = checksum self.__writePrimary(pfd, proot, pkg) self.__writeFilelists(ffd, froot, pkg) # self.__writeOther(ofd, oroot, pkg) try:
import sys, time from pyrpm.database.jointdb import JointDB from pyrpm.database.sqliterepodb import SqliteRepoDB from pyrpm.logger import log sys.path.append("/usr/share/rhn/") try: import up2date_client.up2dateAuth as up2dateAuth import up2date_client.config as rhnconfig from up2date_client import rhnChannel from up2date_client import rhnPackageInfo from up2date_client import up2dateErrors use_rhn = True except: log.warning("Couldn't import up2date_client modules. Disabling RHN support.") use_rhn = False class RhnRepoDB(JointDB): def __init__(self, config, source, buildroot='', nc=None): JointDB.__init__(self, config, source, buildroot) self.comps = None self.reponame = "rhnrepo" if not use_rhn: return up2date_cfg = rhnconfig.initUp2dateConfig() try: login_info = up2dateAuth.getLoginInfo() except up2dateErrors.RhnServerException, e: