Ejemplo n.º 1
0
def process_packages(report, host):
    """ Processes the quoted packages string sent with a report
    """
    if report.packages:
        package_ids = []

        packages = parse_packages(report.packages)
        progress_info_s.send(sender=None,
                             ptext='{0!s} packages'.format(str(host)[0:25]),
                             plen=len(packages))
        for i, pkg_str in enumerate(packages):
            package = process_package(pkg_str, report.protocol)
            if package:
                package_ids.append(package.id)
                try:
                    with transaction.atomic():
                        host.packages.add(package)
                except IntegrityError as e:
                    error_message.send(sender=None, text=e)
                except DatabaseError as e:
                    error_message.send(sender=None, text=e)
            else:
                if pkg_str[0].lower() != 'gpg-pubkey':
                    text = 'No package returned for {0!s}'.format(pkg_str)
                    info_message.send(sender=None, text=text)
            progress_update_s.send(sender=None, index=i + 1)

        for package in host.packages.all():
            if package.id not in package_ids:
                host.packages.remove(package)
Ejemplo n.º 2
0
def process_repos(report, host):
    """ Processes the quoted repos string sent with a report
    """
    if report.repos:
        repo_ids = []
        host_repos = HostRepo.objects.filter(host=host)
        repos = parse_repos(report.repos)

        progress_info_s.send(sender=None,
                             ptext='{0!s} repos'.format(str(host)[0:25]),
                             plen=len(repos))
        for i, repo_str in enumerate(repos):
            repo, priority = process_repo(repo_str, report.arch)
            if repo:
                repo_ids.append(repo.id)
                try:
                    with transaction.atomic():
                        hostrepo, c = host_repos.get_or_create(host=host,
                                                               repo=repo)
                except IntegrityError as e:
                    error_message.send(sender=None, text=e)
                    hostrepo = host_repos.get(host=host, repo=repo)
                try:
                    if hostrepo.priority != priority:
                        hostrepo.priority = priority
                        with transaction.atomic():
                            hostrepo.save()
                except IntegrityError as e:
                    error_message.send(sender=None, text=e)
            progress_update_s.send(sender=None, index=i + 1)

        for hostrepo in host_repos:
            if hostrepo.repo.id not in repo_ids:
                hostrepo.delete()
Ejemplo n.º 3
0
def extract_yast_packages(data):
    """ Extract package metadata from yast metadata file
    """

    extracted = extract(data, 'gz')
    pkgs = re.findall(b'=Pkg: (.*)', extracted)
    plen = len(pkgs)
    packages = set()

    if plen > 0:
        ptext = 'Extracting packages: '
        progress_info_s.send(sender=None, ptext=ptext, plen=plen)

        for i, pkg in enumerate(pkgs):
            progress_update_s.send(sender=None, index=i + 1)
            name, version, release, arch = str(pkg).split()
            package = PackageString(name=name.lower(),
                                    epoch='',
                                    version=version,
                                    release=release,
                                    arch=arch,
                                    packagetype='R')
            packages.add(package)
    else:
        info_message.send(sender=None, text='No packages found in repo')
    return packages
Ejemplo n.º 4
0
def extract_yast_packages(data):
    """ Extract package metadata from yast metadata file
    """

    extracted = extract(data, 'gz')
    pkgs = re.findall(b'=Pkg: (.*)', extracted)
    plen = len(pkgs)
    packages = set()

    if plen > 0:
        ptext = 'Extracting packages: '
        progress_info_s.send(sender=None, ptext=ptext, plen=plen)

        for i, pkg in enumerate(pkgs):
            progress_update_s.send(sender=None, index=i + 1)
            name, version, release, arch = str(pkg).split()
            package = PackageString(name=name.lower(),
                                    epoch='',
                                    version=version,
                                    release=release,
                                    arch=arch,
                                    packagetype='R')
            packages.add(package)
    else:
        info_message.send(sender=None, text='No packages found in repo')
    return packages
Ejemplo n.º 5
0
def extract_deb_packages(data, packages):
    """ Extract package metadata from debian Packages file
    """

    extracted = extract(data)
    package_re = re.compile('^Package: ', re.M)
    plen = len(package_re.findall(extracted))

    if plen > 0:
        ptext = 'Extracting packages: '
        progress_info_s.send(sender=None, ptext=ptext, plen=plen)

        sio = StringIO(extracted)
        for i, stanza in enumerate(Sources.iter_paragraphs(sio)):
            fullversion = Version(stanza['version'])
            arch = stanza['architecture']
            name = stanza['package']
            epoch = fullversion._BaseVersion__epoch
            if epoch is None:
                epoch = ''
            version = fullversion._BaseVersion__upstream_version
            release = fullversion._BaseVersion__debian_revision
            if release is None:
                release = ''
            progress_update_s.send(sender=None, index=i + 1)
            package = PackageString(name=name,
                                    epoch=epoch,
                                    version=version,
                                    release=release,
                                    arch=arch,
                                    packagetype='D')
            packages.add(package)
    else:
        info_message.send(sender=None, text='No packages found in repo\n')
Ejemplo n.º 6
0
def process_packages(report, host):
    """ Processes the quoted packages string sent with a report
    """
    if report.packages:
        old_packages = host.packages.all()
        package_ids = []

        packages = parse_packages(report.packages)
        progress_info_s.send(sender=None,
                             ptext='{0!s} packages'.format(str(host)[0:25]),
                             plen=len(packages))
        for i, pkg_str in enumerate(packages):
            package = process_package(pkg_str, report.protocol)
            if package:
                package_ids.append(package.id)
                try:
                    with transaction.atomic():
                        host.packages.add(package)
                except IntegrityError as e:
                    error_message.send(sender=None, text=e)
                except DatabaseError as e:
                    error_message.send(sender=None, text=e)
            else:
                if pkg_str[0].lower() != 'gpg-pubkey':
                    text = 'No package returned for {0!s}'.format(pkg_str)
                    info_message.send(sender=None, text=text)
            progress_update_s.send(sender=None, index=i + 1)

        removals = old_packages.exclude(pk__in=package_ids)
        for package in removals:
            host.packages.remove(package)
Ejemplo n.º 7
0
def mark_errata_security_updates():
    """ For each set of erratum packages, modify any PackageUpdate that
        should be marked as a security update.
    """
    package_updates = PackageUpdate.objects.all()
    errata = Erratum.objects.all()
    elen = Erratum.objects.count()
    ptext = 'Scanning {0!s} Errata:'.format(elen)
    progress_info_s.send(sender=None, ptext=ptext, plen=elen)
    for i, erratum in enumerate(errata):
        progress_update_s.send(sender=None, index=i + 1)
        if erratum.etype == 'security':
            for package in erratum.packages.all():
                affected_updates = package_updates.filter(
                    newpackage=package,
                    security=False
                )
                for affected_update in affected_updates:
                    if not affected_update.security:
                        affected_update.security = True
                        try:
                            with transaction.atomic():
                                affected_update.save()
                        except IntegrityError as e:
                            error_message.send(sender=None, text=e)
                            # a version of this update already exists that is
                            # marked as a security update, so delete this one
                            affected_update.delete()
Ejemplo n.º 8
0
def process_repos(report, host):
    """ Processes the quoted repos string sent with a report
    """
    if report.repos:
        repo_ids = []
        host_repos = HostRepo.objects.filter(host=host)
        repos = parse_repos(report.repos)

        progress_info_s.send(sender=None,
                             ptext='{0!s} repos'.format(str(host)[0:25]),
                             plen=len(repos))
        for i, repo_str in enumerate(repos):
            repo, priority = process_repo(repo_str, report.arch)
            if repo:
                repo_ids.append(repo.id)
                try:
                    with transaction.atomic():
                        hostrepo, c = host_repos.get_or_create(host=host,
                                                               repo=repo)
                except IntegrityError as e:
                    error_message.send(sender=None, text=e)
                    hostrepo = host_repos.get(host=host, repo=repo)
                try:
                    if hostrepo.priority != priority:
                        hostrepo.priority = priority
                        with transaction.atomic():
                            hostrepo.save()
                except IntegrityError as e:
                    error_message.send(sender=None, text=e)
            progress_update_s.send(sender=None, index=i + 1)

        for hostrepo in host_repos:
            if hostrepo.repo.id not in repo_ids:
                hostrepo.delete()
Ejemplo n.º 9
0
def process_packages(report, host):
    """ Processes the quoted packages string sent with a report """

    if report.packages:
        old_packages = host.packages.all()
        package_ids = []

        packages = parse_packages(report.packages)
        progress_info_s.send(sender=None,
                             ptext='%s packages' % host.__unicode__()[0:25],
                             plen=len(packages))
        for i, pkg_str in enumerate(packages):
            package = process_package(pkg_str, report.protocol)
            if package:
                package_ids.append(package.id)
                try:
                    with transaction.atomic():
                        host.packages.add(package)
                except IntegrityError as e:
                    print e
                except DatabaseError as e:
                    print e
            else:
                print 'No package returned for %s' % pkg_str
            progress_update_s.send(sender=None, index=i + 1)

        removals = old_packages.exclude(pk__in=package_ids)
        for package in removals:
            host.packages.remove(package)
Ejemplo n.º 10
0
def add_updates(updates, host):
    """ Add updates to a Host
    """
    ulen = len(updates)
    if ulen > 0:
        ptext = '{0!s} updates'.format(str(host)[0:25])
        progress_info_s.send(sender=None, ptext=ptext, plen=ulen)

        for i, (u, sec) in enumerate(updates.items):
            update = process_update(host, u, sec)
            if update:
                host.updates.add(update)
            progress_update_s.send(sender=None, index=i + 1)
Ejemplo n.º 11
0
def parse_errata(data, force):
    """ Parse CentOS errata from https://cefs.steve-meier.de/
    """
    result = etree.XML(data)
    errata_xml = result.findall('*')
    elen = len(errata_xml)
    ptext = 'Processing {0!s} Errata:'.format(elen)
    progress_info_s.send(sender=None, ptext=ptext, plen=elen)
    for i, child in enumerate(errata_xml):
        progress_update_s.send(sender=None, index=i + 1)
        if not check_centos_release(child.findall('os_release')):
            continue
        e = parse_errata_tag(child.tag, child.attrib, force)
        if e is not None:
            parse_errata_children(e, child.getchildren())
Ejemplo n.º 12
0
def parse_errata(data, force):
    """ Parse CentOS errata from https://cefs.steve-meier.de/
    """
    result = etree.XML(data)
    errata_xml = result.findall('*')
    elen = len(errata_xml)
    ptext = 'Processing {0!s} Errata:'.format(elen)
    progress_info_s.send(sender=None, ptext=ptext, plen=elen)
    for i, child in enumerate(errata_xml):
        progress_update_s.send(sender=None, index=i + 1)
        if not check_centos_release(child.findall('os_release')):
            continue
        e = parse_errata_tag(child.tag, child.attrib, force)
        if e is not None:
            parse_errata_children(e, child.getchildren())
Ejemplo n.º 13
0
def extract_arch_packages(data):
    """ Extract package metadata from an arch linux tarfile
    """
    from packages.utils import find_evr
    extracted = BytesIO(extract(data, 'gz'))
    tf = tarfile.open(fileobj=extracted, mode='r:*')
    packages = set()
    plen = len(tf.getnames())
    if plen > 0:
        ptext = 'Extracting packages: '
        progress_info_s.send(sender=None, ptext=ptext, plen=plen)
        for i, tarinfo in enumerate(tf):
            progress_update_s.send(sender=None, index=i + 1)
            if tarinfo.isfile():
                name_sec = ver_sec = arch_sec = False
                t = tf.extractfile(tarinfo).read()
                for line in t.decode('utf-8').splitlines():
                    if line.startswith('%NAME%'):
                        name_sec = True
                        continue
                    if name_sec:
                        name_sec = False
                        name = line
                        continue
                    if line.startswith('%VERSION%'):
                        ver_sec = True
                        continue
                    if ver_sec:
                        ver_sec = False
                        epoch, version, release = find_evr(line)
                        continue
                    if line.startswith('%ARCH%'):
                        arch_sec = True
                        continue
                    if arch_sec:
                        arch_sec = False
                        arch = line
                        continue
                package = PackageString(name=name.lower(),
                                        epoch=epoch,
                                        version=version,
                                        release=release,
                                        arch=arch,
                                        packagetype='A')
                packages.add(package)
    else:
        info_message.send(sender=None, text='No packages found in repo')
    return packages
Ejemplo n.º 14
0
def add_updates(updates, host, security):

    ulen = len(updates)
    if security:
        extra = 'sec'
    else:
        extra = 'bug'

    if ulen > 0:
        ptext = '%s %s updates' % (host.__unicode__()[0:25], extra)
        progress_info_s.send(sender=None, ptext=ptext, plen=ulen)
        for i, u in enumerate(updates):
            update = process_update(host, u, security)
            if update:
                host.updates.add(update)
            progress_update_s.send(sender=None, index=i + 1)
Ejemplo n.º 15
0
def add_updates(updates, host, security):

    ulen = len(updates)
    if security:
        extra = 'sec'
    else:
        extra = 'bug'

    if ulen > 0:
        ptext = '{0!s} {1!s} updates'.format(str(host)[0:25], extra)
        progress_info_s.send(sender=None, ptext=ptext, plen=ulen)
        for i, u in enumerate(updates):
            update = process_update(host, u, security)
            if update:
                host.updates.add(update)
            progress_update_s.send(sender=None, index=i + 1)
Ejemplo n.º 16
0
def extract_yum_packages(data, url):
    """ Extract package metadata from a yum primary.xml file
    """

    extracted = extract(data, url)
    ns = 'http://linux.duke.edu/metadata/common'
    context = etree.iterparse(BytesIO(extracted),
                              tag='{{{0!s}}}metadata'.format(ns))
    plen = int(next(context)[1].get('packages'))
    context = etree.iterparse(BytesIO(extracted),
                              tag='{{{0!s}}}package'.format(ns))
    packages = set()

    if plen > 0:
        ptext = 'Extracting packages: '
        progress_info_s.send(sender=None, ptext=ptext, plen=plen)

        for i, data in enumerate(context):
            elem = data[1]
            progress_update_s.send(sender=None, index=i + 1)
            name = elem.xpath('//ns:name',
                              namespaces={'ns': ns})[0].text.lower()
            arch = elem.xpath('//ns:arch',
                              namespaces={'ns': ns})[0].text
            fullversion = elem.xpath('//ns:version',
                                     namespaces={'ns': ns})[0]
            epoch = fullversion.get('epoch')
            version = fullversion.get('ver')
            release = fullversion.get('rel')
            elem.clear()
            while elem.getprevious() is not None:
                del elem.getparent()[0]

            if name != '' and version != '' and arch != '':
                if epoch == '0':
                    epoch = ''
                package = PackageString(name=name,
                                        epoch=epoch,
                                        version=version,
                                        release=release,
                                        arch=arch,
                                        packagetype='R')
                packages.add(package)
    else:
        info_message.send(sender=None, text='No packages found in repo')
    return packages
Ejemplo n.º 17
0
def extract_yum_packages(data, url):
    """ Extract package metadata from a yum primary.xml file
    """

    extracted = extract(data, url)
    ns = 'http://linux.duke.edu/metadata/common'
    context = etree.iterparse(BytesIO(extracted),
                              tag='{{{0!s}}}metadata'.format(ns))
    plen = int(next(context)[1].get('packages'))
    context = etree.iterparse(BytesIO(extracted),
                              tag='{{{0!s}}}package'.format(ns))
    packages = set()

    if plen > 0:
        ptext = 'Extracting packages: '
        progress_info_s.send(sender=None, ptext=ptext, plen=plen)

        for i, data in enumerate(context):
            elem = data[1]
            progress_update_s.send(sender=None, index=i + 1)
            name = elem.xpath('//ns:name',
                              namespaces={'ns': ns})[0].text.lower()
            arch = elem.xpath('//ns:arch',
                              namespaces={'ns': ns})[0].text
            fullversion = elem.xpath('//ns:version',
                                     namespaces={'ns': ns})[0]
            epoch = fullversion.get('epoch')
            version = fullversion.get('ver')
            release = fullversion.get('rel')
            elem.clear()
            while elem.getprevious() is not None:
                del elem.getparent()[0]

            if name != '' and version != '' and arch != '':
                if epoch == '0':
                    epoch = ''
                package = PackageString(name=name,
                                        epoch=epoch,
                                        version=version,
                                        release=release,
                                        arch=arch,
                                        packagetype='R')
                packages.add(package)
    else:
        info_message.send(sender=None, text='No packages found in repo')
    return packages
Ejemplo n.º 18
0
def add_updates(updates, host, security):
    """ Add updates to a Host
    """
    ulen = len(updates)
    if security:
        extra = 'sec'
    else:
        extra = 'bug'

    if ulen > 0:
        ptext = '{0!s} {1!s} updates'.format(str(host)[0:25], extra)
        progress_info_s.send(sender=None, ptext=ptext, plen=ulen)
        for i, u in enumerate(updates):
            update = process_update(host, u, security)
            if update:
                host.updates.add(update)
            progress_update_s.send(sender=None, index=i + 1)
Ejemplo n.º 19
0
def mark_security_updates():
    """ For each set of erratum packages, modify any PackageUpdate that
        should be marked as a security update.
    """
    package_updates = PackageUpdate.objects.all()
    errata = Erratum.objects.all()
    elen = Erratum.objects.count()
    ptext = 'Scanning {0!s} Errata:'.format(elen)
    progress_info_s.send(sender=None, ptext=ptext, plen=elen)
    for i, erratum in enumerate(errata):
        progress_update_s.send(sender=None, index=i + 1)
        if erratum.etype == 'security':
            for package in erratum.packages.all():
                with transaction.atomic():
                    affected_updates = package_updates.select_for_update().filter(newpackage=package)
                    for affected_update in affected_updates:
                        affected_update.security = True
                        affected_update.save()
Ejemplo n.º 20
0
def mark_security_updates():
    """ For each set of erratum packages, modify any PackageUpdate that
        should be marked as a security update.
    """
    package_updates = PackageUpdate.objects.all()
    errata = Erratum.objects.all()
    elen = Erratum.objects.count()
    ptext = 'Scanning {0!s} Errata:'.format(elen)
    progress_info_s.send(sender=None, ptext=ptext, plen=elen)
    for i, erratum in enumerate(errata):
        progress_update_s.send(sender=None, index=i + 1)
        if erratum.etype == 'security':
            for package in erratum.packages.all():
                with transaction.atomic():
                    affected_updates = package_updates.select_for_update(
                    ).filter(newpackage=package)
                    for affected_update in affected_updates:
                        affected_update.security = True
                        affected_update.save()
Ejemplo n.º 21
0
def remove_reports(host, timestamp):
    """ Remove all but the last 3 reports for a host
    """

    from patchman.reports.models import Report

    reports = Report.objects.filter(host=host).order_by('-created')[:3]
    report_ids = []

    for report in reports:
        report_ids.append(report.id)
        report.accessed = timestamp
        report.save()

    del_reports = Report.objects.filter(host=host).exclude(id__in=report_ids)

    rlen = del_reports.count()
    progress_info_s.send(sender=None, ptext='Cleaning %s old reports' % rlen, plen=rlen)
    for i, report in enumerate(del_reports):
        report.delete()
        progress_update_s.send(sender=None, index=i + 1)
Ejemplo n.º 22
0
def extract_deb_packages(data, url):
    """ Extract package metadata from debian Packages file
    """

    extracted = extract(data, url)
    package_re = re.compile(b'^Package: ', re.M)
    plen = len(package_re.findall(extracted))
    packages = set()

    if plen > 0:
        ptext = 'Extracting packages: '
        progress_info_s.send(sender=None, ptext=ptext, plen=plen)

        bio = BytesIO(extracted)
        for i, stanza in enumerate(Packages.iter_paragraphs(bio)):
            # https://github.com/furlongm/patchman/issues/55
            if 'version' not in stanza:
                continue
            fullversion = Version(stanza['version'])
            arch = stanza['architecture']
            name = stanza['package']
            epoch = fullversion._BaseVersion__epoch
            if epoch is None:
                epoch = ''
            version = fullversion._BaseVersion__upstream_version
            release = fullversion._BaseVersion__debian_revision
            if release is None:
                release = ''
            progress_update_s.send(sender=None, index=i + 1)
            package = PackageString(name=name,
                                    epoch=epoch,
                                    version=version,
                                    release=release,
                                    arch=arch,
                                    packagetype='D')
            packages.add(package)
    else:
        info_message.send(sender=None, text='No packages found in repo')
    return packages
Ejemplo n.º 23
0
def remove_reports(host, timestamp):
    """ Remove all but the last 3 reports for a host
    """

    from reports.models import Report

    reports = Report.objects.filter(host=host).order_by('-created')[:3]
    report_ids = []

    for report in reports:
        report_ids.append(report.id)
        report.accessed = timestamp
        report.save()

    del_reports = Report.objects.filter(host=host).exclude(id__in=report_ids)

    rlen = del_reports.count()
    ptext = 'Cleaning {0!s} old reports'.format(rlen)
    progress_info_s.send(sender=None, ptext=ptext, plen=rlen)
    for i, report in enumerate(del_reports):
        report.delete()
        progress_update_s.send(sender=None, index=i + 1)
Ejemplo n.º 24
0
def extract_deb_packages(data, url):
    """ Extract package metadata from debian Packages file
    """

    extracted = extract(data, url)
    package_re = re.compile(b'^Package: ', re.M)
    plen = len(package_re.findall(extracted))
    packages = set()

    if plen > 0:
        ptext = 'Extracting packages: '
        progress_info_s.send(sender=None, ptext=ptext, plen=plen)

        bio = BytesIO(extracted)
        for i, stanza in enumerate(Packages.iter_paragraphs(bio)):
            # https://github.com/furlongm/patchman/issues/55
            if 'version' not in stanza:
                continue
            fullversion = Version(stanza['version'])
            arch = stanza['architecture']
            name = stanza['package']
            epoch = fullversion._BaseVersion__epoch
            if epoch is None:
                epoch = ''
            version = fullversion._BaseVersion__upstream_version
            release = fullversion._BaseVersion__debian_revision
            if release is None:
                release = ''
            progress_update_s.send(sender=None, index=i + 1)
            package = PackageString(name=name,
                                    epoch=epoch,
                                    version=version,
                                    release=release,
                                    arch=arch,
                                    packagetype='D')
            packages.add(package)
    else:
        info_message.send(sender=None, text='No packages found in repo')
    return packages
Ejemplo n.º 25
0
def update_mirror_packages(mirror, packages):
    """ Updates the packages contained on a mirror, and
        removes obsolete packages.
    """

    new = set()
    old = set()
    removed = set()

    repopackages = mirror.packages.all()
    rplen = repopackages.count()

    progress_info_s.send(sender=None, ptext='Obtaining stored packages: ', plen=rplen)
    for i, package in enumerate(repopackages):
        progress_update_s.send(sender=None, index=i + 1)
        name = str(package.name)
        arch = str(package.arch)
        strpackage = PackageString(name=name, epoch=package.epoch, version=package.version, release=package.release, arch=arch, packagetype=package.packagetype)
        old.add(strpackage)

    new = packages.difference(old)
    removed = old.difference(packages)

    nlen = len(new)
    rlen = len(removed)

    progress_info_s.send(sender=None, ptext='Removing %s obsolete packages:' % rlen, plen=rlen)
    for i, package in enumerate(removed):
        progress_update_s.send(sender=None, index=i + 1)
        package_id = PackageName.objects.get(name=package.name)
        epoch = package.epoch
        version = package.version
        release = package.release
        arch = PackageArchitecture.objects.get(name=package.arch)
        packagetype = package.packagetype
        p = Package.objects.get(name=package_id, epoch=epoch, version=version, arch=arch, release=release, packagetype=packagetype)
        from patchman.repos.models import MirrorPackage
        MirrorPackage.objects.get(mirror=mirror, package=p).delete()
    mirror.save()

    progress_info_s.send(sender=None, ptext='Adding %s new packages:' % nlen, plen=nlen)
    for i, package in enumerate(new):
        progress_update_s.send(sender=None, index=i + 1)
        package_id, c = PackageName.objects.get_or_create(name=package.name)
        epoch = package.epoch
        version = package.version
        release = package.release
        packagetype = package.packagetype
        arch, c = PackageArchitecture.objects.get_or_create(name=package.arch)
        p, c = Package.objects.get_or_create(name=package_id, epoch=epoch, version=version, arch=arch, release=release, packagetype=packagetype)
        # This fixes a subtle bug where a stored package name with uppercase letters
        # will not match until it is lowercased.
        if package_id.name != package.name:
            package_id.name = package.name
            package_id.save()
        from patchman.repos.models import MirrorPackage
        MirrorPackage.objects.create(mirror=mirror, package=p)
    mirror.save()
Ejemplo n.º 26
0
def process_repos(report, host):
    """ Processes the quoted repos string sent with a report """

    if report.repos:
        old_repos = host.repos.all()
        repo_ids = []

        host_repos = HostRepo.objects.all()

        repos = parse_repos(report.repos)
        progress_info_s.send(sender=None,
                             ptext='%s repos' % host.__unicode__()[0:25],
                             plen=len(repos))
        for i, repo_str in enumerate(repos):
            repo, priority = process_repo(repo_str, report.arch)
            if repo:
                repo_ids.append(repo.id)
                try:
                    with transaction.atomic():
                        hostrepo, c = host_repos.get_or_create(host=host,
                                                               repo=repo)
                except IntegrityError as e:
                    print e
                    hostrepo = host_repos.get(host=host, repo=repo)
                try:
                    if hostrepo.priority != priority:
                        hostrepo.priority = priority
                        with transaction.atomic():
                            hostrepo.save()
                except IntegrityError as e:
                    print e
            progress_update_s.send(sender=None, index=i + 1)

        removals = old_repos.exclude(pk__in=repo_ids)
        for repo in removals:
            repo.delete()
Ejemplo n.º 27
0
def update_mirror_packages(mirror, packages):
    """ Updates the packages contained on a mirror, and
        removes obsolete packages.
    """
    new = set()
    old = set()
    removals = set()

    mirror_packages = mirror.packages.all()
    mlen = mirror_packages.count()

    ptext = 'Obtaining stored packages: '
    progress_info_s.send(sender=None, ptext=ptext, plen=mlen)
    for i, package in enumerate(mirror_packages):
        progress_update_s.send(sender=None, index=i + 1)
        name = str(package.name)
        arch = str(package.arch)
        strpackage = PackageString(name=name,
                                   epoch=package.epoch,
                                   version=package.version,
                                   release=package.release,
                                   arch=arch,
                                   packagetype=package.packagetype)
        old.add(strpackage)

    new = packages.difference(old)
    removals = old.difference(packages)

    nlen = len(new)
    rlen = len(removals)

    ptext = 'Removing {0!s} obsolete packages:'.format(rlen)
    progress_info_s.send(sender=None, ptext=ptext, plen=rlen)
    for i, package in enumerate(removals):
        progress_update_s.send(sender=None, index=i + 1)
        package_id = PackageName.objects.get(name=package.name)
        epoch = package.epoch
        version = package.version
        release = package.release
        arch = PackageArchitecture.objects.get(name=package.arch)
        packagetype = package.packagetype
        p = Package.objects.get(name=package_id,
                                epoch=epoch,
                                version=version,
                                arch=arch,
                                release=release,
                                packagetype=packagetype)
        from repos.models import MirrorPackage
        with transaction.atomic():
            MirrorPackage.objects.get(mirror=mirror, package=p).delete()

    ptext = 'Adding {0!s} new packages:'.format(nlen)
    progress_info_s.send(sender=None, ptext=ptext, plen=nlen)
    for i, package in enumerate(new):
        progress_update_s.send(sender=None, index=i + 1)

        package_names = PackageName.objects.all()
        with transaction.atomic():
            package_id, c = package_names.get_or_create(name=package.name)

        epoch = package.epoch
        version = package.version
        release = package.release
        packagetype = package.packagetype

        package_arches = PackageArchitecture.objects.all()
        with transaction.atomic():
            arch, c = package_arches.get_or_create(name=package.arch)

        all_packages = Package.objects.all()
        with transaction.atomic():
            p, c = all_packages.get_or_create(name=package_id,
                                              epoch=epoch,
                                              version=version,
                                              arch=arch,
                                              release=release,
                                              packagetype=packagetype)
        # This fixes a subtle bug where a stored package name with uppercase
        # letters will not match until it is lowercased.
        if package_id.name != package.name:
            package_id.name = package.name
            with transaction.atomic():
                package_id.save()
        from repos.models import MirrorPackage  # noqa
        with transaction.atomic():
            MirrorPackage.objects.create(mirror=mirror, package=p)