def add(self, obj, mdtype='updateinfo'): """ Parse a metadata from a given YumRepository, file, or filename. """ if not obj: raise UpdateNoticeException if type(obj) in (type(''), type(u'')): unfile = decompress(obj) infile = open(unfile, 'rt') elif isinstance(obj, YumRepository): if obj.id not in self._repos: self._repos.append(obj.id) md = obj.retrieveMD(mdtype) if not md: raise UpdateNoticeException() unfile = decompress(md) infile = open(unfile, 'rt') elif isinstance(obj, FakeRepository): raise Errors.RepoMDError, "No updateinfo for local pkg" else: # obj is a file object infile = obj for event, elem in iterparse(infile): if elem.tag == 'update': try: un = UpdateNotice(elem) except UpdateNoticeException, e: print >> sys.stderr, "An update notice is broken, skipping." # what else should we do? continue self.add_notice(un)
def add(self, obj, mdtype='updateinfo', all=False): """ Parse a metadata from a given YumRepository, file, or filename. """ if not obj: raise UpdateNoticeException if type(obj) in (type(''), type(u'')): infile = fileutils.decompress_open(obj) elif isinstance(obj, YumRepository): if obj.id not in self._repos: self._repos.append(obj.id) md = obj.retrieveMD(mdtype) if not md: raise UpdateNoticeException() infile = fileutils.decompress_open(md) else: # obj is a file object infile = obj for _event, elem in iterparse(infile): if elem.tag == 'update': un = UpdateNotice(elem) key = un['update_id'] if all: key = "%s-%s" % (un['update_id'], un['version']) if not self._notices.has_key(key): self._notices[key] = un for pkg in un['pkglist']: for pkgfile in pkg['packages']: self._cache['%s-%s-%s' % (pkgfile['name'], pkgfile['version'], pkgfile['release'])] = un no = self._no_cache.setdefault( pkgfile['name'], set()) no.add(un)
def get_metadata_paths(self): def get_location(data_item): for sub_item in data_item: if sub_item.tag.endswith("location"): return sub_item.attrib.get("href") def get_checksum(data_item): for sub_item in data_item: if sub_item.tag.endswith("checksum"): return sub_item.attrib.get("type"), sub_item.text repomd_path = os.path.join(self.repo.basecachedir, self.name, "repomd.xml") if not os.path.isfile(repomd_path): raise RepoMDNotFound(repomd_path) repomd = open(repomd_path, 'rb') files = {} for _event, elem in iterparse(repomd): if elem.tag.endswith("data"): if elem.attrib.get("type") == "primary_db": files['primary'] = (get_location(elem), get_checksum(elem)) elif elem.attrib.get( "type") == "primary" and 'primary' not in files: files['primary'] = (get_location(elem), get_checksum(elem)) elif elem.attrib.get("type") == "updateinfo": files['updateinfo'] = (get_location(elem), get_checksum(elem)) elif elem.attrib.get("type") == "group_gz": files['group'] = (get_location(elem), get_checksum(elem)) elif elem.attrib.get( "type") == "group" and 'group' not in files: files['group'] = (get_location(elem), get_checksum(elem)) repomd.close() return files.values()
def add(self, srcfile = None): if not srcfile: raise CompsException if type(srcfile) in types.StringTypes: # srcfile is a filename string infile = open(srcfile, 'rt') else: # srcfile is a file object infile = srcfile self.compscount += 1 self.compiled = False parser = iterparse(infile) try: for event, elem in parser: if elem.tag == "group": group = Group(elem) self.add_group(group) if elem.tag == "category": category = Category(elem) self.add_category(category) except SyntaxError, e: raise CompsException, "comps file is empty/damaged"
def add(self, obj, mdtype='updateinfo', all_versions=False): """ Parse a metadata from a given YumRepository, file, or filename. """ if not obj: raise UpdateNoticeException if isinstance(obj, (type(''), type(u''))): infile = fileutils.decompress_open(obj) elif isinstance(obj, YumRepository): if obj.id not in self._repos: self._repos.append(obj.id) md = obj.retrieveMD(mdtype) if not md: raise UpdateNoticeException() infile = fileutils.decompress_open(md) else: # obj is a file object infile = obj for _event, elem in iterparse(infile): if elem.tag == 'update': un = UpdateNotice(elem) key = un['update_id'] if all_versions: key = "%s-%s" % (un['update_id'], un['version']) if key not in self._notices: self._notices[key] = un for pkg in un['pkglist']: for pkgfile in pkg['packages']: self._cache['%s-%s-%s' % (pkgfile['name'], pkgfile['version'], pkgfile['release'])] = un no = self._no_cache.setdefault(pkgfile['name'], set()) no.add(un)
def get_metadata_paths(self): def get_location(data_item): for sub_item in data_item: if sub_item.tag.endswith("location"): return sub_item.attrib.get("href") def get_checksum(data_item): for sub_item in data_item: if sub_item.tag.endswith("checksum"): return sub_item.attrib.get("type"), sub_item.text repomd_path = os.path.join(self.repo.basecachedir, self.name, "repomd.xml") if not os.path.isfile(repomd_path): raise RepoMDNotFound(repomd_path) repomd = open(repomd_path, 'rb') files = {} for _event, elem in iterparse(repomd): if elem.tag.endswith("data"): if elem.attrib.get("type") == "primary_db": files['primary'] = (get_location(elem), get_checksum(elem)) elif elem.attrib.get("type") == "primary" and 'primary' not in files: files['primary'] = (get_location(elem), get_checksum(elem)) elif elem.attrib.get("type") == "updateinfo": files['updateinfo'] = (get_location(elem), get_checksum(elem)) elif elem.attrib.get("type") == "group_gz": files['group'] = (get_location(elem), get_checksum(elem)) elif elem.attrib.get("type") == "group" and 'group' not in files: files['group'] = (get_location(elem), get_checksum(elem)) repomd.close() return files.values()
def parse(self, srcfile): if type(srcfile) in (str, ): # srcfile is a filename string try: infile = open(srcfile, 'rt') except IOError: raise RepoMDError("Unable to open %s" % (srcfile, )) else: # srcfile is a file object infile = srcfile srcfile = None # We trust any of these to mean the repomd.xml is valid. infile = AutoFileChecksums(infile, ['sha256', 'sha512'], ignore_missing=True, ignore_none=True) parser = iterparse(infile) try: for event, elem in parser: elem_name = ns_cleanup(elem.tag) if elem_name == "data": thisdata = RepoData(elem=elem) old = self.repoData.get(thisdata.type) if (old and old.size and old.size < thisdata.size and old.location[1].rsplit('.', 1)[1] in _available_compression and srcfile and stat_f( srcfile.rsplit('/', 1)[0] + '/' + thisdata.location[1].rsplit('/', 1)[1]) is None): # previous is smaller, can unzip it, and next is not cached thisdata = old self.repoData[thisdata.type] = thisdata try: # NOTE: This will fail on float timestamps, this is # required for compatibility. Fix is to not generate # float timestamps in repomd.xml. nts = int(thisdata.timestamp) if nts > self.timestamp: # max() not in old python self.timestamp = nts except: pass elif elem_name == "revision": self.revision = elem.text elif elem_name == "tags": for child in elem: child_name = ns_cleanup(child.tag) if child_name == 'content': self.tags['content'].add(child.text) if child_name == 'distro': cpeid = child.attrib.get('cpeid', '') distro = self.tags['distro'].setdefault( cpeid, set()) distro.add(child.text) self.checksums = infile.checksums.hexdigests() self.length = len(infile.checksums) except SyntaxError as e: raise RepoMDError("Damaged repomd.xml file")
def safe_iterparse(filename): """ Works like iterparse, but hides XML errors (prints a warning). """ try: for event, elem in iterparse(filename): yield event, elem except SyntaxError: # Bad XML print >> sys.stderr, "File is not valid XML:", filename
def add(self, obj, mdtype='updateinfo'): """ Parse a metadata from a given YumRepository, file, or filename. """ if not obj: raise UpdateNoticeException if type(obj) in (type(''), type(u'')): unfile = decompress(obj) infile = open(unfile, 'rt') elif isinstance(obj, YumRepository): if obj.id not in self._repos: self._repos.append(obj.id) md = obj.retrieveMD(mdtype) if not md: raise UpdateNoticeException() unfile = repo_gen_decompress(md, 'updateinfo.xml') infile = open(unfile, 'rt') elif isinstance(obj, FakeRepository): raise Errors.RepoMDError, "No updateinfo for local pkg" else: # obj is a file object infile = obj for event, elem in iterparse(infile): if elem.tag == 'update': try: un = UpdateNotice(elem) except UpdateNoticeException, e: print >> sys.stderr, "An update notice is broken, skipping." # what else should we do? continue self.add_notice(un)
def parse(self, srcfile): if type(srcfile) in types.StringTypes: # srcfile is a filename string try: infile = open(srcfile, 'rt') except IOError: raise RepoMDError, "Unable to open %s" %(srcfile,) else: # srcfile is a file object infile = srcfile srcfile = None # We trust any of these to mean the repomd.xml is valid. infile = AutoFileChecksums(infile, ['sha256', 'sha512'], ignore_missing=True, ignore_none=True) parser = iterparse(infile) try: for event, elem in parser: elem_name = ns_cleanup(elem.tag) if elem_name == "data": thisdata = RepoData(elem=elem) old = self.repoData.get(thisdata.type) if (old and old.size and old.size < thisdata.size and old.location[1].rsplit('.', 1)[1] in _available_compression and srcfile and stat_f(srcfile.rsplit('/', 1)[0] +'/'+ thisdata.location[1].rsplit('/', 1)[1]) is None): # previous is smaller, can unzip it, and next is not cached thisdata = old self.repoData[thisdata.type] = thisdata try: # NOTE: This will fail on float timestamps, this is # required for compatability. Fix is to not generate # float timestamps in repomd.xml. nts = int(thisdata.timestamp) if nts > self.timestamp: # max() not in old python self.timestamp = nts except: pass elif elem_name == "revision": self.revision = elem.text elif elem_name == "tags": for child in elem: child_name = ns_cleanup(child.tag) if child_name == 'content': self.tags['content'].add(child.text) if child_name == 'distro': cpeid = child.attrib.get('cpeid', '') distro = self.tags['distro'].setdefault(cpeid,set()) distro.add(child.text) self.checksums = infile.checksums.hexdigests() self.length = len(infile.checksums) except SyntaxError, e: raise RepoMDError, "Damaged repomd.xml file"
def safe_iterparse(filename, logger=None): """ Works like iterparse, but hides XML errors (prints a warning). """ try: for event, elem in iterparse(filename): yield event, elem except SyntaxError: # Bad XML if logger: logger.critical(_("Updateinfo file is not valid XML: %s"), filename) else: print >> sys.stderr, "Updateinfo file is not valid XML:", filename
def parse(self, srcfile): if type(srcfile) in types.StringTypes: # srcfile is a filename string try: infile = open(srcfile, 'rt') except IOError: raise RepoMDError, "Unable to open %s" % (srcfile, ) else: # srcfile is a file object infile = srcfile # We trust any of these to mean the repomd.xml is valid. infile = AutoFileChecksums(infile, ['sha256', 'sha512'], ignore_missing=True, ignore_none=True) parser = iterparse(infile) try: for event, elem in parser: elem_name = ns_cleanup(elem.tag) if elem_name == "data": thisdata = RepoData(elem=elem) self.repoData[thisdata.type] = thisdata try: nts = int(thisdata.timestamp) if nts > self.timestamp: # max() not in old python self.timestamp = nts except: pass elif elem_name == "revision": self.revision = elem.text elif elem_name == "tags": for child in elem: child_name = ns_cleanup(child.tag) if child_name == 'content': self.tags['content'].add(child.text) if child_name == 'distro': cpeid = child.attrib.get('cpeid', '') distro = self.tags['distro'].setdefault( cpeid, set()) distro.add(child.text) self.checksums = infile.checksums.hexdigests() self.length = len(infile.checksums) except SyntaxError, e: raise RepoMDError, "Damaged repomd.xml file"
def parse(self, srcfile): if type(srcfile) in types.StringTypes: # srcfile is a filename string try: infile = open(srcfile, 'rt') except IOError: raise RepoMDError, "Unable to open %s" %(srcfile,) else: # srcfile is a file object infile = srcfile # We trust any of these to mean the repomd.xml is valid. infile = AutoFileChecksums(infile, ['sha256', 'sha512'], ignore_missing=True, ignore_none=True) parser = iterparse(infile) try: for event, elem in parser: elem_name = ns_cleanup(elem.tag) if elem_name == "data": thisdata = RepoData(elem=elem) self.repoData[thisdata.type] = thisdata try: nts = int(thisdata.timestamp) if nts > self.timestamp: # max() not in old python self.timestamp = nts except: pass elif elem_name == "revision": self.revision = elem.text elif elem_name == "tags": for child in elem: child_name = ns_cleanup(child.tag) if child_name == 'content': self.tags['content'].add(child.text) if child_name == 'distro': cpeid = child.attrib.get('cpeid', '') distro = self.tags['distro'].setdefault(cpeid,set()) distro.add(child.text) self.checksums = infile.checksums.hexdigests() self.length = len(infile.checksums) except SyntaxError, e: raise RepoMDError, "Damaged repomd.xml file"
def add(self, srcfile=None): if not srcfile: raise CompsException if type(srcfile) in (str, ): # srcfile is a filename string try: infile = open(srcfile, 'rt') except IOError as e: raise CompsException('open(%s): #%u %s' % (srcfile, e.errno, e.strerror)) else: # srcfile is a file object infile = srcfile self.compscount += 1 self.compiled = False parser = iterparse(infile) try: for event, elem in parser: if elem.tag == "group": group = Group(elem) self.add_group(group) if elem.tag == "environment": environment = Environment(elem) self.add_environment(environment) if elem.tag == "category": category = Category(elem) self.add_category(category) if elem.tag == "langpacks": self._langpacks.parse(elem) except SyntaxError as e: raise CompsException("comps file is empty/damaged") del parser
if type(srcfile) in types.StringTypes: # srcfile is a filename string try: infile = open(srcfile, 'rt') except IOError, e: raise CompsException, 'open(%s): #%u %s' % (srcfile, e.errno, e.strerror) else: # srcfile is a file object infile = srcfile self.compscount += 1 self.compiled = False parser = iterparse(infile) try: for event, elem in parser: if elem.tag == "group": group = Group(elem) self.add_group(group) if elem.tag == "environment": environment = Environment(elem) self.add_environment(environment) if elem.tag == "category": category = Category(elem) self.add_category(category) if elem.tag == "langpacks": self._langpacks.parse(elem) except SyntaxError, e: raise CompsException, "comps file is empty/damaged"
raise CompsException if type(srcfile) in types.StringTypes: # srcfile is a filename string try: infile = open(srcfile, 'rt') except IOError, e: raise CompsException, 'open(%s): #%u %s' % (srcfile, e.errno, e.strerror) else: # srcfile is a file object infile = srcfile self.compscount += 1 self.compiled = False parser = iterparse(infile) try: for event, elem in parser: if elem.tag == "group": group = Group(elem) self.add_group(group) if elem.tag == "environment": environment = Environment(elem) self.add_environment(environment) if elem.tag == "category": category = Category(elem) self.add_category(category) if elem.tag == "langpacks": self._langpacks.parse(elem) except SyntaxError, e: raise CompsException, "comps file is empty/damaged"