def extract_yast_packages(data):
""" Extract package metadata from yast metadata file
"""
extracted = extract(data, 'gz')
pkgs = re.findall(b'=Pkg: (.*)', extracted)
plen = len(pkgs)
packages = set()
if plen > 0:
ptext = 'Extracting packages: '
progress_info_s.send(sender=None, ptext=ptext, plen=plen)
for i, pkg in enumerate(pkgs):
progress_update_s.send(sender=None, index=i + 1)
name, version, release, arch = str(pkg).split()
package = PackageString(name=name.lower(),
epoch='',
version=version,
release=release,
arch=arch,
packagetype='R')
packages.add(package)
else:
info_message.send(sender=None, text='No packages found in repo')
return packages
def extract_yum_packages(data, url):
""" Extract package metadata from a yum primary.xml file
"""
extracted = extract(data, url)
ns = 'http://linux.duke.edu/metadata/common'
context = etree.iterparse(BytesIO(extracted),
tag='{{{0!s}}}metadata'.format(ns))
plen = int(next(context)[1].get('packages'))
context = etree.iterparse(BytesIO(extracted),
tag='{{{0!s}}}package'.format(ns))
packages = set()
if plen > 0:
ptext = 'Extracting packages: '
progress_info_s.send(sender=None, ptext=ptext, plen=plen)
for i, data in enumerate(context):
elem = data[1]
progress_update_s.send(sender=None, index=i + 1)
name = elem.xpath('//ns:name',
namespaces={'ns': ns})[0].text.lower()
arch = elem.xpath('//ns:arch',
namespaces={'ns': ns})[0].text
fullversion = elem.xpath('//ns:version',
namespaces={'ns': ns})[0]
epoch = fullversion.get('epoch')
version = fullversion.get('ver')
release = fullversion.get('rel')
elem.clear()
while elem.getprevious() is not None:
del elem.getparent()[0]
if name != '' and version != '' and arch != '':
if epoch == '0':
epoch = ''
package = PackageString(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch,
packagetype='R')
packages.add(package)
else:
info_message.send(sender=None, text='No packages found in repo')
return packages
def extract_deb_packages(data, url):
""" Extract package metadata from debian Packages file
"""
extracted = extract(data, url)
package_re = re.compile(b'^Package: ', re.M)
plen = len(package_re.findall(extracted))
packages = set()
if plen > 0:
ptext = 'Extracting packages: '
progress_info_s.send(sender=None, ptext=ptext, plen=plen)
bio = BytesIO(extracted)
for i, stanza in enumerate(Packages.iter_paragraphs(bio)):
# https://github.com/furlongm/patchman/issues/55
if 'version' not in stanza:
continue
fullversion = Version(stanza['version'])
arch = stanza['architecture']
name = stanza['package']
epoch = fullversion._BaseVersion__epoch
if epoch is None:
epoch = ''
version = fullversion._BaseVersion__upstream_version
release = fullversion._BaseVersion__debian_revision
if release is None:
release = ''
progress_update_s.send(sender=None, index=i + 1)
package = PackageString(name=name,
epoch=epoch,
version=version,
release=release,
arch=arch,
packagetype='D')
packages.add(package)
else:
info_message.send(sender=None, text='No packages found in repo')
return packages
def update_mirror_packages(mirror, packages):
""" Updates the packages contained on a mirror, and
removes obsolete packages.
"""
new = set()
old = set()
removals = set()
mirror_packages = mirror.packages.all()
mlen = mirror_packages.count()
ptext = 'Obtaining stored packages: '
progress_info_s.send(sender=None, ptext=ptext, plen=mlen)
for i, package in enumerate(mirror_packages):
progress_update_s.send(sender=None, index=i + 1)
name = str(package.name)
arch = str(package.arch)
strpackage = PackageString(name=name,
epoch=package.epoch,
version=package.version,
release=package.release,
arch=arch,
packagetype=package.packagetype)
old.add(strpackage)
new = packages.difference(old)
removals = old.difference(packages)
nlen = len(new)
rlen = len(removals)
ptext = 'Removing {0!s} obsolete packages:'.format(rlen)
progress_info_s.send(sender=None, ptext=ptext, plen=rlen)
for i, package in enumerate(removals):
progress_update_s.send(sender=None, index=i + 1)
package_id = PackageName.objects.get(name=package.name)
epoch = package.epoch
version = package.version
release = package.release
arch = PackageArchitecture.objects.get(name=package.arch)
packagetype = package.packagetype
p = Package.objects.get(name=package_id,
epoch=epoch,
version=version,
arch=arch,
release=release,
packagetype=packagetype)
from patchman.repos.models import MirrorPackage
with transaction.atomic():
MirrorPackage.objects.get(mirror=mirror, package=p).delete()
ptext = 'Adding {0!s} new packages:'.format(nlen)
progress_info_s.send(sender=None, ptext=ptext, plen=nlen)
for i, package in enumerate(new):
progress_update_s.send(sender=None, index=i + 1)
package_names = PackageName.objects.all()
with transaction.atomic():
package_id, c = package_names.get_or_create(name=package.name)
epoch = package.epoch
version = package.version
release = package.release
packagetype = package.packagetype
package_arches = PackageArchitecture.objects.all()
with transaction.atomic():
arch, c = package_arches.get_or_create(name=package.arch)
all_packages = Package.objects.all()
with transaction.atomic():
p, c = all_packages.get_or_create(name=package_id,
epoch=epoch,
version=version,
arch=arch,
release=release,
packagetype=packagetype)
# This fixes a subtle bug where a stored package name with uppercase
# letters will not match until it is lowercased.
if package_id.name != package.name:
package_id.name = package.name
with transaction.atomic():
package_id.save()
from patchman.repos.models import MirrorPackage
with transaction.atomic():
MirrorPackage.objects.create(mirror=mirror, package=p)