def regen_rrds(logger=None):
"""
Regenerates the rrd database
"""
if logger is None:
logger = FakeLogger()
logger.info("Regenering RRDs for world")
for wlog in WorldLog.objects.all():
charts.rrd_update('world', wlog.datetime, wlog)
logger.info("Regenering RRDs for categories")
for clog in CategoryLog.objects.all():
charts.rrd_update('category-%s' % clog.category,
clog.datetime, clog)
logger.info("Regenering RRDs for herds")
for hlog in HerdLog.objects.all():
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
logger.info("Regenering RRDs for maintainers")
for mlog in MaintainerLog.objects.all():
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
mlog.datetime, mlog)
def stabilization_candidates(days_to_candidate=30, logger=None):
"""
Collect stabilization candidates
"""
if logger is None:
logger = FakeLogger()
date_diff = datetime.utcnow() - timedelta(days=days_to_candidate)
date_limit = date_diff.date()
logger.info("Collecting stabilization candidates - date_limit=%s", str(date_limit))
# Set all versions to not be stabilization_candidates
# Version.objects.update(stabilization_candidate=False)
# For every version check if it's unstable.
# If it is then check if can be a stabilization candidate
versions = Version.objects.filter(overlay="gentoo").filter(Q(vtype="release") | Q(vtype="p"))
for version in versions:
pkg = Package(version.cpv())
try:
keywords = pkg.environment("KEYWORDS").split()
except GentoolkitFatalError:
logger.warning("Error while processing %s!", version)
continue
if all([x.startswith("~") for x in keywords]):
version_date = get_version_date(version, date_limit)
if version_date:
logger.info("+ [s] %s @ %s", version, version_date)
# XXX: What should we save? A flag and the date?Just the date?
version.stabilization_candidate = version_date
version.save()
logger.info("Finished collecting stabilization candidates")
def scan_portage(packages=None,
category=None,
no_log=False,
upstream=False,
purge_packages=False,
purge_versions=False,
logger=None):
logger = logger or FakeLogger()
scan_handler = ScanPortage(logger=logger,
no_log=no_log,
purge_packages=purge_packages,
purge_versions=purge_versions)
logger.info('Scanning portage tree...')
if not packages:
qs = Package.objects.all()
if category:
qs = qs.filter(category=category)
prefetch_packages = qs
else:
results = []
for package in packages:
if isinstance(package, Package):
results.append(package)
else:
if '/' in package:
cat, pkg = portage.catsplit(package)
qs = Package.objects.filter(category=cat, name=pkg)
else:
qs = Package.objects.filter(name=package)
for package in qs:
results.append(package)
prefetch_packages = results
scan_handler.prefetch(prefetch_packages, category)
if not packages and category:
scan_handler.scan(category=category)
elif not packages:
scan_handler.scan()
else:
for pkg in packages:
if isinstance(pkg, Package):
scan_handler.scan('%s/%s' % (pkg.category, pkg.name))
else:
scan_handler.scan(pkg)
populate_categories(logger)
populate_overlays(logger)
logger.info('Done.')
return scan_handler.packages_updated()
def update_portage_trees(logger=None):
from djeuscan.processing import FakeLogger
logger = logger or FakeLogger()
logger.info("Running emerge --sync")
emerge_sync(logger)
emerge_metadata(logger)
logger.info("Running layman --sync")
layman_sync(logger, cache=True)
#logger.info("Running emerge --regen")
#emerge_regen(logger)
logger.info("Running eix-update")
eix_update(logger)
logger.info("Done!")
def stabilization_candidates(days_to_candidate=30, logger=None):
"""
Collect stabilization candidates
"""
if logger is None:
logger = FakeLogger()
date_diff = (datetime.utcnow() - timedelta(days=days_to_candidate))
date_limit = date_diff.date()
logger.info("Collecting stabilization candidates - date_limit=%s",
str(date_limit))
# Set all versions to not be stabilization_candidates
#Version.objects.update(stabilization_candidate=False)
# For every version check if it's unstable.
# If it is then check if can be a stabilization candidate
versions = Version.objects.filter(overlay='gentoo').filter(
Q(vtype='release') | Q(vtype='p'))
for version in versions:
pkg = Package(version.cpv())
try:
keywords = pkg.environment("KEYWORDS").split()
except GentoolkitFatalError:
logger.warning("Error while processing %s!", version)
continue
if all([x.startswith("~") for x in keywords]):
version_date = get_version_date(version, date_limit)
if version_date:
logger.info('+ [s] %s @ %s', version, version_date)
# XXX: What should we save? A flag and the date?Just the date?
version.stabilization_candidate = version_date
version.save()
logger.info("Finished collecting stabilization candidates")
def __init__(self,
logger=None,
no_log=False,
purge_packages=False,
purge_versions=False,
upstream=False):
self.logger = logger or FakeLogger()
self.no_log = no_log
self.purge_packages = purge_packages
self.purge_versions = purge_versions
self.upstream = upstream
self.style = color_style()
self._cache = {'packages': {}, 'versions': {}}
self._overlays = None
self._packages_updated = set()
self._versions = set()
self._versions_seen = set()
def scan_upstream(packages=None, purge_versions=False, logger=None):
logger = logger or FakeLogger()
scan_handler = ScanUpstream(logger=logger, purge_versions=purge_versions)
logger.info('Scanning upstream...')
if not packages:
packages = Package.objects.all()
for pkg in packages:
try:
package = '%s/%s' % (pkg.category, pkg.name)
except AttributeError:
package = pkg
logger.info('Scanning %s' % package)
scan_handler.scan(package)
scan_handler.purge_old_versions()
logger.info('Done.')
def __init__(self, logger=None, purge_versions=False):
self.logger = logger or FakeLogger()
self.purge_versions = purge_versions
self._versions = set()
self._versions_seen = set()
def __init__(self, logger=None):
self.style = color_style()
self.logger = logger or FakeLogger()
def update_counters(fast=False, nolog=False, logger=None):
logger = logger or FakeLogger()
now = timezone.now()
categories = {}
herds = {}
maintainers = {}
wlog = None
if not nolog:
wlog = WorldLog()
wlog.datetime = now
for cat in Package.objects.values('category').distinct():
clog = CategoryLog()
clog.datetime = now
clog.category = cat['category']
categories[clog.category] = clog
for herd in Herd.objects.all():
hlog = HerdLog()
hlog.datetime = now
hlog.herd = herd
herds[herd.id] = hlog
for maintainer in Maintainer.objects.all():
mlog = MaintainerLog()
mlog.datetime = now
mlog.maintainer = maintainer
maintainers[maintainer.id] = mlog
package_queryset = Package.objects.all()
n_versions = {}
n_packaged = {}
n_overlay = {}
last_versions_gentoo = {}
last_versions_overlay = {}
last_versions_upstream = {}
if not fast:
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
for version in Version.objects.all().values(*attrs):
overlay, packaged = version['overlay'], version['packaged']
package_id = version['package_id']
_add_safe(n_versions, package_id)
if not packaged:
_add_last_ver(last_versions_upstream, version)
continue
if overlay == 'gentoo':
_add_safe(n_packaged, package_id)
_add_last_ver(last_versions_gentoo, version)
else:
_add_safe(n_overlay, package_id)
_add_last_ver(last_versions_overlay, version)
for package in package_queryset.select_related('herds', 'maintainers'):
if not fast:
package.n_versions = n_versions.get(package.id, 0)
package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0)
default = {'id': None}
package.last_version_gentoo_id = last_versions_gentoo.get(
package.id, default)['id']
package.last_version_overlay_id = last_versions_overlay.get(
package.id, default)['id']
package.last_version_upstream_id = last_versions_upstream.get(
package.id, default)['id']
package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged \
+ package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay \
< package.n_versions)
def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
def update_log(storage, qs):
for row in qs:
update_row(storage, row['id'])
if not nolog:
update_log(herds, package.herds.all().values('id'))
update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category)
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
if nolog:
return
for clog in categories.values():
logger.info('+ [cl] %s', clog)
charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save()
for hlog in herds.values():
logger.info('+ [hl] %s', hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save()
for mlog in maintainers.values():
logger.info('+ [ml] %s', mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()
charts.rrd_update('world', now, wlog)
wlog.save()