def extract_yast_packages(data): """ Extract package metadata from yast metadata file """ extracted = extract(data, 'gz') pkgs = re.findall(b'=Pkg: (.*)', extracted) plen = len(pkgs) packages = set() if plen > 0: ptext = 'Extracting packages: ' progress_info_s.send(sender=None, ptext=ptext, plen=plen) for i, pkg in enumerate(pkgs): progress_update_s.send(sender=None, index=i + 1) name, version, release, arch = str(pkg).split() package = PackageString(name=name.lower(), epoch='', version=version, release=release, arch=arch, packagetype='R') packages.add(package) else: info_message.send(sender=None, text='No packages found in repo') return packages
def extract_yum_packages(data, url): """ Extract package metadata from a yum primary.xml file """ extracted = extract(data, url) ns = 'http://linux.duke.edu/metadata/common' context = etree.iterparse(BytesIO(extracted), tag='{{{0!s}}}metadata'.format(ns)) plen = int(next(context)[1].get('packages')) context = etree.iterparse(BytesIO(extracted), tag='{{{0!s}}}package'.format(ns)) packages = set() if plen > 0: ptext = 'Extracting packages: ' progress_info_s.send(sender=None, ptext=ptext, plen=plen) for i, data in enumerate(context): elem = data[1] progress_update_s.send(sender=None, index=i + 1) name = elem.xpath('//ns:name', namespaces={'ns': ns})[0].text.lower() arch = elem.xpath('//ns:arch', namespaces={'ns': ns})[0].text fullversion = elem.xpath('//ns:version', namespaces={'ns': ns})[0] epoch = fullversion.get('epoch') version = fullversion.get('ver') release = fullversion.get('rel') elem.clear() while elem.getprevious() is not None: del elem.getparent()[0] if name != '' and version != '' and arch != '': if epoch == '0': epoch = '' package = PackageString(name=name, epoch=epoch, version=version, release=release, arch=arch, packagetype='R') packages.add(package) else: info_message.send(sender=None, text='No packages found in repo') return packages
def get_metalink_urls(url): """ Parses a metalink and returns a list of mirrors """ res = get_url(url) if response_is_valid(res): if 'content-type' in res.headers and \ res.headers['content-type'] == 'application/metalink+xml': data = download_url(res, 'Downloading repo info:') ns = 'http://www.metalinker.org/' try: context = etree.parse(BytesIO(data), etree.XMLParser()) except etree.XMLSyntaxError: context = etree.parse(BytesIO(extract(data, 'gz')), etree.XMLParser()) xpath = "//ns:files/ns:file[@name='repomd.xml']/ns:resources/ns:url[@protocol='https']" # noqa metalink_urls = context.xpath(xpath, namespaces={'ns': ns}) return [x.text for x in metalink_urls]
def get_primary_url(mirror_url, data): if isinstance(data, text_type): if data.startswith('Bad repo - not in list') or \ data.startswith('Invalid repo'): return None, None, None ns = 'http://linux.duke.edu/metadata/repo' try: context = etree.parse(BytesIO(data), etree.XMLParser()) except etree.XMLSyntaxError: context = etree.parse(BytesIO(extract(data, 'gz')), etree.XMLParser()) location = context.xpath("//ns:data[@type='primary']/ns:location/@href", namespaces={'ns': ns})[0] checksum = context.xpath("//ns:data[@type='primary']/ns:checksum", namespaces={'ns': ns})[0].text csum_type = context.xpath("//ns:data[@type='primary']/ns:checksum/@type", namespaces={'ns': ns})[0] primary_url = str(mirror_url.rsplit('/', 2)[0]) + '/' + location return primary_url, checksum, csum_type
def extract_deb_packages(data, url): """ Extract package metadata from debian Packages file """ extracted = extract(data, url) package_re = re.compile(b'^Package: ', re.M) plen = len(package_re.findall(extracted)) packages = set() if plen > 0: ptext = 'Extracting packages: ' progress_info_s.send(sender=None, ptext=ptext, plen=plen) bio = BytesIO(extracted) for i, stanza in enumerate(Packages.iter_paragraphs(bio)): # https://github.com/furlongm/patchman/issues/55 if 'version' not in stanza: continue fullversion = Version(stanza['version']) arch = stanza['architecture'] name = stanza['package'] epoch = fullversion._BaseVersion__epoch if epoch is None: epoch = '' version = fullversion._BaseVersion__upstream_version release = fullversion._BaseVersion__debian_revision if release is None: release = '' progress_update_s.send(sender=None, index=i + 1) package = PackageString(name=name, epoch=epoch, version=version, release=release, arch=arch, packagetype='D') packages.add(package) else: info_message.send(sender=None, text='No packages found in repo') return packages