示例#1
0
def regen_rrds(logger=None):
    """
    Regenerates the rrd database
    """

    if logger is None:
        logger = FakeLogger()

    logger.info("Regenering RRDs for world")
    for wlog in WorldLog.objects.all():
        charts.rrd_update('world', wlog.datetime, wlog)

    logger.info("Regenering RRDs for categories")
    for clog in CategoryLog.objects.all():
        charts.rrd_update('category-%s' % clog.category,
                          clog.datetime, clog)

    logger.info("Regenering RRDs for herds")
    for hlog in HerdLog.objects.all():
        charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)

    logger.info("Regenering RRDs for maintainers")
    for mlog in MaintainerLog.objects.all():
        charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
                          mlog.datetime, mlog)
示例#2
0
def stabilization_candidates(days_to_candidate=30, logger=None):
    """
    Collect stabilization candidates
    """

    if logger is None:
        logger = FakeLogger()

    date_diff = datetime.utcnow() - timedelta(days=days_to_candidate)
    date_limit = date_diff.date()

    logger.info("Collecting stabilization candidates - date_limit=%s", str(date_limit))

    # Set all versions to not be stabilization_candidates
    # Version.objects.update(stabilization_candidate=False)

    # For every version check if it's unstable.
    # If it is then check if can be a stabilization candidate
    versions = Version.objects.filter(overlay="gentoo").filter(Q(vtype="release") | Q(vtype="p"))
    for version in versions:
        pkg = Package(version.cpv())
        try:
            keywords = pkg.environment("KEYWORDS").split()
        except GentoolkitFatalError:
            logger.warning("Error while processing %s!", version)
            continue
        if all([x.startswith("~") for x in keywords]):
            version_date = get_version_date(version, date_limit)
            if version_date:
                logger.info("+ [s] %s @ %s", version, version_date)
                # XXX: What should we save? A flag and the date?Just the date?
                version.stabilization_candidate = version_date
                version.save()

    logger.info("Finished collecting stabilization candidates")
示例#3
0
def scan_portage(packages=None,
                 category=None,
                 no_log=False,
                 upstream=False,
                 purge_packages=False,
                 purge_versions=False,
                 logger=None):

    logger = logger or FakeLogger()

    scan_handler = ScanPortage(logger=logger,
                               no_log=no_log,
                               purge_packages=purge_packages,
                               purge_versions=purge_versions)

    logger.info('Scanning portage tree...')

    if not packages:
        qs = Package.objects.all()
        if category:
            qs = qs.filter(category=category)
        prefetch_packages = qs
    else:
        results = []
        for package in packages:
            if isinstance(package, Package):
                results.append(package)
            else:
                if '/' in package:
                    cat, pkg = portage.catsplit(package)
                    qs = Package.objects.filter(category=cat, name=pkg)
                else:
                    qs = Package.objects.filter(name=package)
                for package in qs:
                    results.append(package)
        prefetch_packages = results

    scan_handler.prefetch(prefetch_packages, category)

    if not packages and category:
        scan_handler.scan(category=category)
    elif not packages:
        scan_handler.scan()
    else:
        for pkg in packages:
            if isinstance(pkg, Package):
                scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
            else:
                scan_handler.scan(pkg)

    populate_categories(logger)
    populate_overlays(logger)

    logger.info('Done.')
    return scan_handler.packages_updated()
示例#4
0
def update_portage_trees(logger=None):
    from djeuscan.processing import FakeLogger

    logger = logger or FakeLogger()
    logger.info("Running emerge --sync")
    emerge_sync(logger)
    emerge_metadata(logger)
    logger.info("Running layman --sync")
    layman_sync(logger, cache=True)
    #logger.info("Running emerge --regen")
    #emerge_regen(logger)
    logger.info("Running eix-update")
    eix_update(logger)
    logger.info("Done!")
示例#5
0
def stabilization_candidates(days_to_candidate=30, logger=None):
    """
    Collect stabilization candidates
    """

    if logger is None:
        logger = FakeLogger()

    date_diff = (datetime.utcnow() - timedelta(days=days_to_candidate))
    date_limit = date_diff.date()

    logger.info("Collecting stabilization candidates - date_limit=%s",
                str(date_limit))

    # Set all versions to not be stabilization_candidates
    #Version.objects.update(stabilization_candidate=False)

    # For every version check if it's unstable.
    # If it is then check if can be a stabilization candidate
    versions = Version.objects.filter(overlay='gentoo').filter(
        Q(vtype='release') | Q(vtype='p'))
    for version in versions:
        pkg = Package(version.cpv())
        try:
            keywords = pkg.environment("KEYWORDS").split()
        except GentoolkitFatalError:
            logger.warning("Error while processing %s!", version)
            continue
        if all([x.startswith("~") for x in keywords]):
            version_date = get_version_date(version, date_limit)
            if version_date:
                logger.info('+ [s] %s @ %s', version, version_date)
                # XXX: What should we save? A flag and the date?Just the date?
                version.stabilization_candidate = version_date
                version.save()

    logger.info("Finished collecting stabilization candidates")
示例#6
0
    def __init__(self,
                 logger=None,
                 no_log=False,
                 purge_packages=False,
                 purge_versions=False,
                 upstream=False):
        self.logger = logger or FakeLogger()
        self.no_log = no_log
        self.purge_packages = purge_packages
        self.purge_versions = purge_versions
        self.upstream = upstream

        self.style = color_style()

        self._cache = {'packages': {}, 'versions': {}}
        self._overlays = None
        self._packages_updated = set()
        self._versions = set()
        self._versions_seen = set()
示例#7
0
def scan_upstream(packages=None, purge_versions=False, logger=None):
    logger = logger or FakeLogger()

    scan_handler = ScanUpstream(logger=logger, purge_versions=purge_versions)

    logger.info('Scanning upstream...')

    if not packages:
        packages = Package.objects.all()

    for pkg in packages:
        try:
            package = '%s/%s' % (pkg.category, pkg.name)
        except AttributeError:
            package = pkg

        logger.info('Scanning %s' % package)
        scan_handler.scan(package)

    scan_handler.purge_old_versions()

    logger.info('Done.')
示例#8
0
 def __init__(self, logger=None, purge_versions=False):
     self.logger = logger or FakeLogger()
     self.purge_versions = purge_versions
     self._versions = set()
     self._versions_seen = set()
示例#9
0
 def __init__(self, logger=None):
     self.style = color_style()
     self.logger = logger or FakeLogger()
示例#10
0
def update_counters(fast=False, nolog=False, logger=None):
    logger = logger or FakeLogger()

    now = timezone.now()

    categories = {}
    herds = {}
    maintainers = {}

    wlog = None

    if not nolog:
        wlog = WorldLog()
        wlog.datetime = now

        for cat in Package.objects.values('category').distinct():
            clog = CategoryLog()
            clog.datetime = now
            clog.category = cat['category']
            categories[clog.category] = clog

        for herd in Herd.objects.all():
            hlog = HerdLog()
            hlog.datetime = now
            hlog.herd = herd
            herds[herd.id] = hlog

        for maintainer in Maintainer.objects.all():
            mlog = MaintainerLog()
            mlog.datetime = now
            mlog.maintainer = maintainer
            maintainers[maintainer.id] = mlog

    package_queryset = Package.objects.all()

    n_versions = {}
    n_packaged = {}
    n_overlay = {}

    last_versions_gentoo = {}
    last_versions_overlay = {}
    last_versions_upstream = {}

    if not fast:
        attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
        for version in Version.objects.all().values(*attrs):
            overlay, packaged = version['overlay'], version['packaged']
            package_id = version['package_id']

            _add_safe(n_versions, package_id)

            if not packaged:
                _add_last_ver(last_versions_upstream, version)
                continue
            if overlay == 'gentoo':
                _add_safe(n_packaged, package_id)
                _add_last_ver(last_versions_gentoo, version)
            else:
                _add_safe(n_overlay, package_id)
                _add_last_ver(last_versions_overlay, version)

    for package in package_queryset.select_related('herds', 'maintainers'):
        if not fast:
            package.n_versions = n_versions.get(package.id, 0)
            package.n_packaged = n_packaged.get(package.id, 0)
            package.n_overlay = n_overlay.get(package.id, 0)

            default = {'id': None}
            package.last_version_gentoo_id = last_versions_gentoo.get(
                package.id, default)['id']
            package.last_version_overlay_id = last_versions_overlay.get(
                package.id, default)['id']
            package.last_version_upstream_id = last_versions_upstream.get(
                package.id, default)['id']
            package.save()

        n_packages_gentoo = int(package.n_packaged == package.n_versions)
        n_packages_overlay = int(package.n_overlay and package.n_packaged \
                                 + package.n_overlay == package.n_versions)
        n_packages_outdated = int(package.n_packaged + package.n_overlay \
                                  < package.n_versions)

        def update_row(storage, key):
            storage[key].n_packages_gentoo += n_packages_gentoo
            storage[key].n_packages_overlay += n_packages_overlay
            storage[key].n_packages_outdated += n_packages_outdated

            storage[key].n_versions_gentoo += package.n_packaged
            storage[key].n_versions_overlay += package.n_overlay
            storage[key].n_versions_upstream += package.n_versions - \
                                                package.n_packaged - \
                                                package.n_overlay

        def update_log(storage, qs):
            for row in qs:
                update_row(storage, row['id'])

        if not nolog:
            update_log(herds, package.herds.all().values('id'))
            update_log(maintainers, package.maintainers.all().values('id'))
            update_row(categories, package.category)

            wlog.n_packages_gentoo += n_packages_gentoo
            wlog.n_packages_overlay += n_packages_overlay
            wlog.n_packages_outdated += n_packages_outdated

            wlog.n_versions_gentoo += package.n_packaged
            wlog.n_versions_overlay += package.n_overlay
            wlog.n_versions_upstream += package.n_versions - \
                                        package.n_packaged - \
                                        package.n_overlay

    if nolog:
        return

    for clog in categories.values():
        logger.info('+ [cl] %s', clog)
        charts.rrd_update('category-%s' % clog.category, now, clog)
        clog.save()

    for hlog in herds.values():
        logger.info('+ [hl] %s', hlog)
        charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
        hlog.save()

    for mlog in maintainers.values():
        logger.info('+ [ml] %s', mlog)
        charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
        mlog.save()

    charts.rrd_update('world', now, wlog)
    wlog.save()