def extract_deb_packages(data, packages): """ Extract package metadata from debian Packages file """ extracted = extract(data) package_re = re.compile('^Package: ', re.M) plen = len(package_re.findall(extracted)) if plen > 0: ptext = 'Extracting packages: ' progress_info_s.send(sender=None, ptext=ptext, plen=plen) sio = StringIO(extracted) for i, stanza in enumerate(Sources.iter_paragraphs(sio)): fullversion = Version(stanza['version']) arch = stanza['architecture'] name = stanza['package'] epoch = fullversion._BaseVersion__epoch if epoch is None: epoch = '' version = fullversion._BaseVersion__upstream_version release = fullversion._BaseVersion__debian_revision if release is None: release = '' progress_update_s.send(sender=None, index=i + 1) package = PackageString(name=name, epoch=epoch, version=version, release=release, arch=arch, packagetype='D') packages.add(package) else: info_message.send(sender=None, text='No packages found in repo\n')
def extract_yast_packages(data): """ Extract package metadata from yast metadata file """ extracted = extract(data, 'gz') pkgs = re.findall(b'=Pkg: (.*)', extracted) plen = len(pkgs) packages = set() if plen > 0: ptext = 'Extracting packages: ' progress_info_s.send(sender=None, ptext=ptext, plen=plen) for i, pkg in enumerate(pkgs): progress_update_s.send(sender=None, index=i + 1) name, version, release, arch = str(pkg).split() package = PackageString(name=name.lower(), epoch='', version=version, release=release, arch=arch, packagetype='R') packages.add(package) else: info_message.send(sender=None, text='No packages found in repo') return packages
def process_packages(report, host): """ Processes the quoted packages string sent with a report """ if report.packages: old_packages = host.packages.all() package_ids = [] packages = parse_packages(report.packages) progress_info_s.send(sender=None, ptext='{0!s} packages'.format(str(host)[0:25]), plen=len(packages)) for i, pkg_str in enumerate(packages): package = process_package(pkg_str, report.protocol) if package: package_ids.append(package.id) try: with transaction.atomic(): host.packages.add(package) except IntegrityError as e: error_message.send(sender=None, text=e) except DatabaseError as e: error_message.send(sender=None, text=e) else: if pkg_str[0].lower() != 'gpg-pubkey': text = 'No package returned for {0!s}'.format(pkg_str) info_message.send(sender=None, text=text) progress_update_s.send(sender=None, index=i + 1) removals = old_packages.exclude(pk__in=package_ids) for package in removals: host.packages.remove(package)
def extract_yast_packages(data): """ Extract package metadata from yast metadata file """ extracted = extract(data, 'gz').decode('utf-8') pkgs = re.findall('=Pkg: (.*)', extracted) plen = len(pkgs) packages = set() if plen > 0: ptext = 'Extracting packages: ' progress_info_s.send(sender=None, ptext=ptext, plen=plen) for i, pkg in enumerate(pkgs): progress_update_s.send(sender=None, index=i + 1) name, version, release, arch = pkg.split() package = PackageString(name=name.lower(), epoch='', version=version, release=release, arch=arch, packagetype='R') packages.add(package) else: info_message.send(sender=None, text='No packages found in repo') return packages
def process_repos(report, host): """ Processes the quoted repos string sent with a report """ if report.repos: repo_ids = [] host_repos = HostRepo.objects.filter(host=host) repos = parse_repos(report.repos) progress_info_s.send(sender=None, ptext='{0!s} repos'.format(str(host)[0:25]), plen=len(repos)) for i, repo_str in enumerate(repos): repo, priority = process_repo(repo_str, report.arch) if repo: repo_ids.append(repo.id) try: with transaction.atomic(): hostrepo, c = host_repos.get_or_create(host=host, repo=repo) except IntegrityError as e: error_message.send(sender=None, text=e) hostrepo = host_repos.get(host=host, repo=repo) try: if hostrepo.priority != priority: hostrepo.priority = priority with transaction.atomic(): hostrepo.save() except IntegrityError as e: error_message.send(sender=None, text=e) progress_update_s.send(sender=None, index=i + 1) for hostrepo in host_repos: if hostrepo.repo.id not in repo_ids: hostrepo.delete()
def process_packages(report, host): """ Processes the quoted packages string sent with a report """ if report.packages: old_packages = host.packages.all() package_ids = [] packages = parse_packages(report.packages) progress_info_s.send(sender=None, ptext='%s packages' % host.__unicode__()[0:25], plen=len(packages)) for i, pkg_str in enumerate(packages): package = process_package(pkg_str, report.protocol) if package: package_ids.append(package.id) try: with transaction.atomic(): host.packages.add(package) except IntegrityError as e: print e except DatabaseError as e: print e else: print 'No package returned for %s' % pkg_str progress_update_s.send(sender=None, index=i + 1) removals = old_packages.exclude(pk__in=package_ids) for package in removals: host.packages.remove(package)
def process_packages(report, host): """ Processes the quoted packages string sent with a report """ if report.packages: package_ids = [] packages = parse_packages(report.packages) progress_info_s.send(sender=None, ptext='{0!s} packages'.format(str(host)[0:25]), plen=len(packages)) for i, pkg_str in enumerate(packages): package = process_package(pkg_str, report.protocol) if package: package_ids.append(package.id) try: with transaction.atomic(): host.packages.add(package) except IntegrityError as e: error_message.send(sender=None, text=e) except DatabaseError as e: error_message.send(sender=None, text=e) else: if pkg_str[0].lower() != 'gpg-pubkey': text = 'No package returned for {0!s}'.format(pkg_str) info_message.send(sender=None, text=text) progress_update_s.send(sender=None, index=i + 1) for package in host.packages.all(): if package.id not in package_ids: host.packages.remove(package)
def mark_errata_security_updates(): """ For each set of erratum packages, modify any PackageUpdate that should be marked as a security update. """ package_updates = PackageUpdate.objects.all() errata = Erratum.objects.all() elen = Erratum.objects.count() ptext = 'Scanning {0!s} Errata:'.format(elen) progress_info_s.send(sender=None, ptext=ptext, plen=elen) for i, erratum in enumerate(errata): progress_update_s.send(sender=None, index=i + 1) if erratum.etype == 'security': for package in erratum.packages.all(): affected_updates = package_updates.filter(newpackage=package, security=False) for affected_update in affected_updates: if not affected_update.security: affected_update.security = True try: with transaction.atomic(): affected_update.save() except IntegrityError as e: error_message.send(sender=None, text=e) # a version of this update already exists that is # marked as a security update, so delete this one affected_update.delete()
def add_updates(updates, host): """ Add updates to a Host """ ulen = len(updates) if ulen > 0: ptext = '{0!s} updates'.format(str(host)[0:25]) progress_info_s.send(sender=None, ptext=ptext, plen=ulen) for i, (u, sec) in enumerate(updates.items): update = process_update(host, u, sec) if update: host.updates.add(update) progress_update_s.send(sender=None, index=i + 1)
def parse_errata(data, force): """ Parse CentOS errata from https://cefs.steve-meier.de/ """ result = etree.XML(data) errata_xml = result.findall('*') elen = len(errata_xml) ptext = 'Processing {0!s} Errata:'.format(elen) progress_info_s.send(sender=None, ptext=ptext, plen=elen) for i, child in enumerate(errata_xml): progress_update_s.send(sender=None, index=i + 1) if not check_centos_release(child.findall('os_release')): continue e = parse_errata_tag(child.tag, child.attrib, force) if e is not None: parse_errata_children(e, child.getchildren())
def extract_arch_packages(data): """ Extract package metadata from an arch linux tarfile """ from packages.utils import find_evr extracted = BytesIO(extract(data, 'gz')) tf = tarfile.open(fileobj=extracted, mode='r:*') packages = set() plen = len(tf.getnames()) if plen > 0: ptext = 'Extracting packages: ' progress_info_s.send(sender=None, ptext=ptext, plen=plen) for i, tarinfo in enumerate(tf): progress_update_s.send(sender=None, index=i + 1) if tarinfo.isfile(): name_sec = ver_sec = arch_sec = False t = tf.extractfile(tarinfo).read() for line in t.decode('utf-8').splitlines(): if line.startswith('%NAME%'): name_sec = True continue if name_sec: name_sec = False name = line continue if line.startswith('%VERSION%'): ver_sec = True continue if ver_sec: ver_sec = False epoch, version, release = find_evr(line) continue if line.startswith('%ARCH%'): arch_sec = True continue if arch_sec: arch_sec = False arch = line continue package = PackageString(name=name.lower(), epoch=epoch, version=version, release=release, arch=arch, packagetype='A') packages.add(package) else: info_message.send(sender=None, text='No packages found in repo') return packages
def add_updates(updates, host, security): ulen = len(updates) if security: extra = 'sec' else: extra = 'bug' if ulen > 0: ptext = '%s %s updates' % (host.__unicode__()[0:25], extra) progress_info_s.send(sender=None, ptext=ptext, plen=ulen) for i, u in enumerate(updates): update = process_update(host, u, security) if update: host.updates.add(update) progress_update_s.send(sender=None, index=i + 1)
def add_updates(updates, host, security): ulen = len(updates) if security: extra = 'sec' else: extra = 'bug' if ulen > 0: ptext = '{0!s} {1!s} updates'.format(str(host)[0:25], extra) progress_info_s.send(sender=None, ptext=ptext, plen=ulen) for i, u in enumerate(updates): update = process_update(host, u, security) if update: host.updates.add(update) progress_update_s.send(sender=None, index=i + 1)
def add_updates(updates, host, security): """ Add updates to a Host """ ulen = len(updates) if security: extra = 'sec' else: extra = 'bug' if ulen > 0: ptext = '{0!s} {1!s} updates'.format(str(host)[0:25], extra) progress_info_s.send(sender=None, ptext=ptext, plen=ulen) for i, u in enumerate(updates): update = process_update(host, u, security) if update: host.updates.add(update) progress_update_s.send(sender=None, index=i + 1)
def extract_yum_packages(data, url): """ Extract package metadata from a yum primary.xml file """ extracted = extract(data, url) ns = 'http://linux.duke.edu/metadata/common' context = etree.iterparse(BytesIO(extracted), tag='{{{0!s}}}metadata'.format(ns)) plen = int(next(context)[1].get('packages')) context = etree.iterparse(BytesIO(extracted), tag='{{{0!s}}}package'.format(ns)) packages = set() if plen > 0: ptext = 'Extracting packages: ' progress_info_s.send(sender=None, ptext=ptext, plen=plen) for i, data in enumerate(context): elem = data[1] progress_update_s.send(sender=None, index=i + 1) name = elem.xpath('//ns:name', namespaces={'ns': ns})[0].text.lower() arch = elem.xpath('//ns:arch', namespaces={'ns': ns})[0].text fullversion = elem.xpath('//ns:version', namespaces={'ns': ns})[0] epoch = fullversion.get('epoch') version = fullversion.get('ver') release = fullversion.get('rel') elem.clear() while elem.getprevious() is not None: del elem.getparent()[0] if name != '' and version != '' and arch != '': if epoch == '0': epoch = '' package = PackageString(name=name, epoch=epoch, version=version, release=release, arch=arch, packagetype='R') packages.add(package) else: info_message.send(sender=None, text='No packages found in repo') return packages
def mark_security_updates(): """ For each set of erratum packages, modify any PackageUpdate that should be marked as a security update. """ package_updates = PackageUpdate.objects.all() errata = Erratum.objects.all() elen = Erratum.objects.count() ptext = 'Scanning {0!s} Errata:'.format(elen) progress_info_s.send(sender=None, ptext=ptext, plen=elen) for i, erratum in enumerate(errata): progress_update_s.send(sender=None, index=i + 1) if erratum.etype == 'security': for package in erratum.packages.all(): with transaction.atomic(): affected_updates = package_updates.select_for_update().filter(newpackage=package) for affected_update in affected_updates: affected_update.security = True affected_update.save()
def mark_security_updates(): """ For each set of erratum packages, modify any PackageUpdate that should be marked as a security update. """ package_updates = PackageUpdate.objects.all() errata = Erratum.objects.all() elen = Erratum.objects.count() ptext = 'Scanning {0!s} Errata:'.format(elen) progress_info_s.send(sender=None, ptext=ptext, plen=elen) for i, erratum in enumerate(errata): progress_update_s.send(sender=None, index=i + 1) if erratum.etype == 'security': for package in erratum.packages.all(): with transaction.atomic(): affected_updates = package_updates.select_for_update( ).filter(newpackage=package) for affected_update in affected_updates: affected_update.security = True affected_update.save()
def remove_reports(host, timestamp): """ Remove all but the last 3 reports for a host """ from patchman.reports.models import Report reports = Report.objects.filter(host=host).order_by('-created')[:3] report_ids = [] for report in reports: report_ids.append(report.id) report.accessed = timestamp report.save() del_reports = Report.objects.filter(host=host).exclude(id__in=report_ids) rlen = del_reports.count() progress_info_s.send(sender=None, ptext='Cleaning %s old reports' % rlen, plen=rlen) for i, report in enumerate(del_reports): report.delete() progress_update_s.send(sender=None, index=i + 1)
def extract_deb_packages(data, url): """ Extract package metadata from debian Packages file """ extracted = extract(data, url) package_re = re.compile(b'^Package: ', re.M) plen = len(package_re.findall(extracted)) packages = set() if plen > 0: ptext = 'Extracting packages: ' progress_info_s.send(sender=None, ptext=ptext, plen=plen) bio = BytesIO(extracted) for i, stanza in enumerate(Packages.iter_paragraphs(bio)): # https://github.com/furlongm/patchman/issues/55 if 'version' not in stanza: continue fullversion = Version(stanza['version']) arch = stanza['architecture'] name = stanza['package'] epoch = fullversion._BaseVersion__epoch if epoch is None: epoch = '' version = fullversion._BaseVersion__upstream_version release = fullversion._BaseVersion__debian_revision if release is None: release = '' progress_update_s.send(sender=None, index=i + 1) package = PackageString(name=name, epoch=epoch, version=version, release=release, arch=arch, packagetype='D') packages.add(package) else: info_message.send(sender=None, text='No packages found in repo') return packages
def remove_reports(host, timestamp): """ Remove all but the last 3 reports for a host """ from reports.models import Report reports = Report.objects.filter(host=host).order_by('-created')[:3] report_ids = [] for report in reports: report_ids.append(report.id) report.accessed = timestamp report.save() del_reports = Report.objects.filter(host=host).exclude(id__in=report_ids) rlen = del_reports.count() ptext = 'Cleaning {0!s} old reports'.format(rlen) progress_info_s.send(sender=None, ptext=ptext, plen=rlen) for i, report in enumerate(del_reports): report.delete() progress_update_s.send(sender=None, index=i + 1)
def update_mirror_packages(mirror, packages): """ Updates the packages contained on a mirror, and removes obsolete packages. """ new = set() old = set() removed = set() repopackages = mirror.packages.all() rplen = repopackages.count() progress_info_s.send(sender=None, ptext='Obtaining stored packages: ', plen=rplen) for i, package in enumerate(repopackages): progress_update_s.send(sender=None, index=i + 1) name = str(package.name) arch = str(package.arch) strpackage = PackageString(name=name, epoch=package.epoch, version=package.version, release=package.release, arch=arch, packagetype=package.packagetype) old.add(strpackage) new = packages.difference(old) removed = old.difference(packages) nlen = len(new) rlen = len(removed) progress_info_s.send(sender=None, ptext='Removing %s obsolete packages:' % rlen, plen=rlen) for i, package in enumerate(removed): progress_update_s.send(sender=None, index=i + 1) package_id = PackageName.objects.get(name=package.name) epoch = package.epoch version = package.version release = package.release arch = PackageArchitecture.objects.get(name=package.arch) packagetype = package.packagetype p = Package.objects.get(name=package_id, epoch=epoch, version=version, arch=arch, release=release, packagetype=packagetype) from patchman.repos.models import MirrorPackage MirrorPackage.objects.get(mirror=mirror, package=p).delete() mirror.save() progress_info_s.send(sender=None, ptext='Adding %s new packages:' % nlen, plen=nlen) for i, package in enumerate(new): progress_update_s.send(sender=None, index=i + 1) package_id, c = PackageName.objects.get_or_create(name=package.name) epoch = package.epoch version = package.version release = package.release packagetype = package.packagetype arch, c = PackageArchitecture.objects.get_or_create(name=package.arch) p, c = Package.objects.get_or_create(name=package_id, epoch=epoch, version=version, arch=arch, release=release, packagetype=packagetype) # This fixes a subtle bug where a stored package name with uppercase letters # will not match until it is lowercased. if package_id.name != package.name: package_id.name = package.name package_id.save() from patchman.repos.models import MirrorPackage MirrorPackage.objects.create(mirror=mirror, package=p) mirror.save()
def process_repos(report, host): """ Processes the quoted repos string sent with a report """ if report.repos: old_repos = host.repos.all() repo_ids = [] host_repos = HostRepo.objects.all() repos = parse_repos(report.repos) progress_info_s.send(sender=None, ptext='%s repos' % host.__unicode__()[0:25], plen=len(repos)) for i, repo_str in enumerate(repos): repo, priority = process_repo(repo_str, report.arch) if repo: repo_ids.append(repo.id) try: with transaction.atomic(): hostrepo, c = host_repos.get_or_create(host=host, repo=repo) except IntegrityError as e: print e hostrepo = host_repos.get(host=host, repo=repo) try: if hostrepo.priority != priority: hostrepo.priority = priority with transaction.atomic(): hostrepo.save() except IntegrityError as e: print e progress_update_s.send(sender=None, index=i + 1) removals = old_repos.exclude(pk__in=repo_ids) for repo in removals: repo.delete()
def update_mirror_packages(mirror, packages): """ Updates the packages contained on a mirror, and removes obsolete packages. """ new = set() old = set() removals = set() mirror_packages = mirror.packages.all() mlen = mirror_packages.count() ptext = 'Obtaining stored packages: ' progress_info_s.send(sender=None, ptext=ptext, plen=mlen) for i, package in enumerate(mirror_packages): progress_update_s.send(sender=None, index=i + 1) name = str(package.name) arch = str(package.arch) strpackage = PackageString(name=name, epoch=package.epoch, version=package.version, release=package.release, arch=arch, packagetype=package.packagetype) old.add(strpackage) new = packages.difference(old) removals = old.difference(packages) nlen = len(new) rlen = len(removals) ptext = 'Removing {0!s} obsolete packages:'.format(rlen) progress_info_s.send(sender=None, ptext=ptext, plen=rlen) for i, package in enumerate(removals): progress_update_s.send(sender=None, index=i + 1) package_id = PackageName.objects.get(name=package.name) epoch = package.epoch version = package.version release = package.release arch = PackageArchitecture.objects.get(name=package.arch) packagetype = package.packagetype p = Package.objects.get(name=package_id, epoch=epoch, version=version, arch=arch, release=release, packagetype=packagetype) from repos.models import MirrorPackage with transaction.atomic(): MirrorPackage.objects.get(mirror=mirror, package=p).delete() ptext = 'Adding {0!s} new packages:'.format(nlen) progress_info_s.send(sender=None, ptext=ptext, plen=nlen) for i, package in enumerate(new): progress_update_s.send(sender=None, index=i + 1) package_names = PackageName.objects.all() with transaction.atomic(): package_id, c = package_names.get_or_create(name=package.name) epoch = package.epoch version = package.version release = package.release packagetype = package.packagetype package_arches = PackageArchitecture.objects.all() with transaction.atomic(): arch, c = package_arches.get_or_create(name=package.arch) all_packages = Package.objects.all() with transaction.atomic(): p, c = all_packages.get_or_create(name=package_id, epoch=epoch, version=version, arch=arch, release=release, packagetype=packagetype) # This fixes a subtle bug where a stored package name with uppercase # letters will not match until it is lowercased. if package_id.name != package.name: package_id.name = package.name with transaction.atomic(): package_id.save() from repos.models import MirrorPackage # noqa with transaction.atomic(): MirrorPackage.objects.create(mirror=mirror, package=p)