def extract_yast_packages(data):
""" Extract package metadata from yast metadata file
"""
extracted = extract(data, 'gz')
pkgs = re.findall(b'=Pkg: (.*)', extracted)
plen = len(pkgs)
packages = set()
if plen > 0:
ptext = 'Extracting packages: '
progress_info_s.send(sender=None, ptext=ptext, plen=plen)
for i, pkg in enumerate(pkgs):
progress_update_s.send(sender=None, index=i + 1)
name, version, release, arch = str(pkg).split()
package = PackageString(name=name.lower(),
epoch='',
version=version,
release=release,
arch=arch,
packagetype='R')
packages.add(package)
else:
info_message.send(sender=None, text='No packages found in repo')
return packages
def extract_yast_packages(data):
""" Extract package metadata from yast metadata file
"""
extracted = extract(data, 'gz')
pkgs = re.findall(b'=Pkg: (.*)', extracted)
plen = len(pkgs)
packages = set()
if plen > 0:
ptext = 'Extracting packages: '
progress_info_s.send(sender=None, ptext=ptext, plen=plen)
for i, pkg in enumerate(pkgs):
progress_update_s.send(sender=None, index=i + 1)
name, version, release, arch = str(pkg).split()
package = PackageString(name=name.lower(),
epoch='',
version=version,
release=release,
arch=arch,
packagetype='R')
packages.add(package)
else:
info_message.send(sender=None, text='No packages found in repo')
return packages
def extract_yum_packages(data, url):
""" Extract package metadata from a yum primary.xml file
"""
extracted = extract(data, url)
ns = 'http://linux.duke.edu/metadata/common'
context = etree.iterparse(BytesIO(extracted),
tag='{{{0!s}}}metadata'.format(ns))
plen = int(next(context)[1].get('packages'))
context = etree.iterparse(BytesIO(extracted),
tag='{{{0!s}}}package'.format(ns))
packages = set()
if plen > 0:
ptext = 'Extracting packages: '
progress_info_s.send(sender=None, ptext=ptext, plen=plen)
for i, data in enumerate(context):
elem = data[1]
progress_update_s.send(sender=None, index=i + 1)
name = elem.xpath('//ns:name',
namespaces={'ns': ns})[0].text.lower()
arch = elem.xpath('//ns:arch',
namespaces={'ns': ns})[0].text
fullversion = elem.xpath('//ns:version',
namespaces={'ns': ns})[0]
epoch = fullversion.get('epoch')
version = fullversion.get('ver')
release = fullversion.get('rel')
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
if name != '' and version != '' and arch != '':
if epoch == '0':
epoch = ''
package = PackageString(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch,
packagetype='R')
packages.add(package)
else:
info_message.send(sender=None, text='No packages found in repo')
return packages
def get_metalink_urls(url):
""" Parses a metalink and returns a list of mirrors
"""
res = get_url(url)
if response_is_valid(res):
if 'content-type' in res.headers and \
res.headers['content-type'] == 'application/metalink+xml':
data = download_url(res, 'Downloading repo info:')
ns = 'http://www.metalinker.org/'
try:
context = etree.parse(BytesIO(data), etree.XMLParser())
except etree.XMLSyntaxError:
context = etree.parse(BytesIO(extract(data, 'gz')),
etree.XMLParser())
xpath = "//ns:files/ns:file[@name='repomd.xml']/ns:resources/ns:url[@protocol='https']" # noqa
metalink_urls = context.xpath(xpath, namespaces={'ns': ns})
return [x.text for x in metalink_urls]
def extract_yum_packages(data, url):
""" Extract package metadata from a yum primary.xml file
"""
extracted = extract(data, url)
ns = 'http://linux.duke.edu/metadata/common'
context = etree.iterparse(BytesIO(extracted),
tag='{{{0!s}}}metadata'.format(ns))
plen = int(next(context)[1].get('packages'))
context = etree.iterparse(BytesIO(extracted),
tag='{{{0!s}}}package'.format(ns))
packages = set()
if plen > 0:
ptext = 'Extracting packages: '
progress_info_s.send(sender=None, ptext=ptext, plen=plen)
for i, data in enumerate(context):
elem = data[1]
progress_update_s.send(sender=None, index=i + 1)
name = elem.xpath('//ns:name',
namespaces={'ns': ns})[0].text.lower()
arch = elem.xpath('//ns:arch',
namespaces={'ns': ns})[0].text
fullversion = elem.xpath('//ns:version',
namespaces={'ns': ns})[0]
epoch = fullversion.get('epoch')
version = fullversion.get('ver')
release = fullversion.get('rel')
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
if name != '' and version != '' and arch != '':
if epoch == '0':
epoch = ''
package = PackageString(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch,
packagetype='R')
packages.add(package)
else:
info_message.send(sender=None, text='No packages found in repo')
return packages
def get_metalink_urls(url):
""" Parses a metalink and returns a list of mirrors
"""
res = get_url(url)
if response_is_valid(res):
if 'content-type' in res.headers and \
res.headers['content-type'] == 'application/metalink+xml':
data = download_url(res, 'Downloading repo info:')
ns = 'http://www.metalinker.org/'
try:
context = etree.parse(BytesIO(data), etree.XMLParser())
except etree.XMLSyntaxError:
context = etree.parse(BytesIO(extract(data, 'gz')),
etree.XMLParser())
xpath = "//ns:files/ns:file[@name='repomd.xml']/ns:resources/ns:url[@protocol='https']" # noqa
metalink_urls = context.xpath(xpath, namespaces={'ns': ns})
return [x.text for x in metalink_urls]
def get_primary_url(mirror_url, data):
if isinstance(data, text_type):
if data.startswith('Bad repo - not in list') or \
data.startswith('Invalid repo'):
return None, None, None
ns = 'http://linux.duke.edu/metadata/repo'
try:
context = etree.parse(BytesIO(data), etree.XMLParser())
except etree.XMLSyntaxError:
context = etree.parse(BytesIO(extract(data, 'gz')), etree.XMLParser())
location = context.xpath("//ns:data[@type='primary']/ns:location/@href",
namespaces={'ns': ns})[0]
checksum = context.xpath("//ns:data[@type='primary']/ns:checksum",
namespaces={'ns': ns})[0].text
csum_type = context.xpath("//ns:data[@type='primary']/ns:checksum/@type",
namespaces={'ns': ns})[0]
primary_url = str(mirror_url.rsplit('/', 2)[0]) + '/' + location
return primary_url, checksum, csum_type
def get_primary_url(mirror_url, data):
if isinstance(data, text_type):
if data.startswith('Bad repo - not in list') or \
data.startswith('Invalid repo'):
return None, None, None
ns = 'http://linux.duke.edu/metadata/repo'
try:
context = etree.parse(BytesIO(data), etree.XMLParser())
except etree.XMLSyntaxError:
context = etree.parse(BytesIO(extract(data, 'gz')), etree.XMLParser())
location = context.xpath("//ns:data[@type='primary']/ns:location/@href",
namespaces={'ns': ns})[0]
checksum = context.xpath("//ns:data[@type='primary']/ns:checksum",
namespaces={'ns': ns})[0].text
csum_type = context.xpath("//ns:data[@type='primary']/ns:checksum/@type",
namespaces={'ns': ns})[0]
primary_url = str(mirror_url.rsplit('/', 2)[0]) + '/' + location
return primary_url, checksum, csum_type
def extract_deb_packages(data, url):
""" Extract package metadata from debian Packages file
"""
extracted = extract(data, url)
package_re = re.compile(b'^Package: ', re.M)
plen = len(package_re.findall(extracted))
packages = set()
if plen > 0:
ptext = 'Extracting packages: '
progress_info_s.send(sender=None, ptext=ptext, plen=plen)
bio = BytesIO(extracted)
for i, stanza in enumerate(Packages.iter_paragraphs(bio)):
# https://github.com/furlongm/patchman/issues/55
if 'version' not in stanza:
continue
fullversion = Version(stanza['version'])
arch = stanza['architecture']
name = stanza['package']
epoch = fullversion._BaseVersion__epoch
if epoch is None:
epoch = ''
version = fullversion._BaseVersion__upstream_version
release = fullversion._BaseVersion__debian_revision
if release is None:
release = ''
progress_update_s.send(sender=None, index=i + 1)
package = PackageString(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch,
packagetype='D')
packages.add(package)
else:
info_message.send(sender=None, text='No packages found in repo')
return packages
def extract_deb_packages(data, url):
""" Extract package metadata from debian Packages file
"""
extracted = extract(data, url)
package_re = re.compile(b'^Package: ', re.M)
plen = len(package_re.findall(extracted))
packages = set()
if plen > 0:
ptext = 'Extracting packages: '
progress_info_s.send(sender=None, ptext=ptext, plen=plen)
bio = BytesIO(extracted)
for i, stanza in enumerate(Packages.iter_paragraphs(bio)):
# https://github.com/furlongm/patchman/issues/55
if 'version' not in stanza:
continue
fullversion = Version(stanza['version'])
arch = stanza['architecture']
name = stanza['package']
epoch = fullversion._BaseVersion__epoch
if epoch is None:
epoch = ''
version = fullversion._BaseVersion__upstream_version
release = fullversion._BaseVersion__debian_revision
if release is None:
release = ''
progress_update_s.send(sender=None, index=i + 1)
package = PackageString(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch,
packagetype='D')
packages.add(package)
else:
info_message.send(sender=None, text='No packages found in repo')
return packages
