def regen_rrds(logger=None):
"""
Regenerates the rrd database
"""
if logger is None:
logger = FakeLogger()
logger.info("Regenering RRDs for world")
for wlog in WorldLog.objects.all():
charts.rrd_update('world', wlog.datetime, wlog)
logger.info("Regenering RRDs for categories")
for clog in CategoryLog.objects.all():
charts.rrd_update('category-%s' % clog.category,
clog.datetime, clog)
logger.info("Regenering RRDs for herds")
for hlog in HerdLog.objects.all():
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
logger.info("Regenering RRDs for maintainers")
for mlog in MaintainerLog.objects.all():
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
mlog.datetime, mlog)
def handle(self, *args, **options):
for wlog in WorldLog.objects.all():
charts.rrd_update('world', wlog.datetime, wlog)
for clog in CategoryLog.objects.all():
charts.rrd_update('category-%s' % clog.category,
clog.datetime, clog)
for hlog in HerdLog.objects.all():
charts.rrd_update('herd-%d' % hlog.herd.id, hlog.datetime, hlog)
for mlog in MaintainerLog.objects.all():
charts.rrd_update('maintainer-%d' % mlog.maintainer.id,
mlog.datetime, mlog)
def handle(self, *args, **options):
now = timezone.now()
categories = {}
herds = {}
maintainers = {}
wlog = None
if not options['nolog']:
wlog = WorldLog()
wlog.datetime = now
for cat in Package.objects.values('category').distinct():
clog = CategoryLog()
clog.datetime = now
clog.category = cat['category']
categories[clog.category] = clog
for herd in Herd.objects.all():
hlog = HerdLog()
hlog.datetime = now
hlog.herd = herd
herds[herd.id] = hlog
for maintainer in Maintainer.objects.all():
mlog = MaintainerLog()
mlog.datetime = now
mlog.maintainer = maintainer
maintainers[maintainer.id] = mlog
package_queryset = Package.objects.all()
n_versions = {}
n_packaged = {}
n_overlay = {}
last_versions_gentoo = {}
last_versions_overlay = {}
last_versions_upstream = {}
def add_safe(storage, key):
if key not in storage:
storage[key] = 1
else:
storage[key] += 1
def add_last_ver(storage, version):
key = version['package_id']
if key not in storage:
storage[key] = version
return
if version['version'].startswith('9999'):
return
if compare_versions(storage[key]['version'],
version['version']) < 0:
storage[key] = version
if not options['fast']:
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
for version in Version.objects.all().values(*attrs):
overlay, packaged = version['overlay'], version['packaged']
package_id = version['package_id']
add_safe(n_versions, package_id)
if not packaged:
add_last_ver(last_versions_upstream, version)
continue
if overlay == 'gentoo':
add_safe(n_packaged, package_id)
add_last_ver(last_versions_gentoo, version)
else:
add_safe(n_overlay, package_id)
add_last_ver(last_versions_overlay, version)
for package in package_queryset.select_related('herds', 'maintainers'):
if not options['fast']:
package.n_versions = n_versions.get(package.id, 0)
package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0)
default = {'id': None}
package.last_version_gentoo_id = last_versions_gentoo.get(
package.id, default
)['id']
package.last_version_overlay_id = last_versions_overlay.get(
package.id, default
)['id']
package.last_version_upstream_id = last_versions_upstream.get(
package.id, default
)['id']
package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged \
+ package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay \
< package.n_versions)
def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
def update_log(storage, qs):
for row in qs:
update_row(storage, row['id'])
if not options['nolog']:
update_log(herds, package.herds.all().values('id'))
update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category)
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
if options['nolog']:
return
for clog in categories.values():
if not options['quiet']:
self.stdout.write('+ [cl] %s\n' % clog)
charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save()
for hlog in herds.values():
if not options['quiet']:
self.stdout.write('+ [hl] %s\n' % hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save()
for mlog in maintainers.values():
if not options['quiet']:
self.stdout.write('+ [ml] %s\n' % mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()
charts.rrd_update('world', now, wlog)
wlog.save()
def update_counters(fast=False, nolog=False, logger=None):
logger = logger or FakeLogger()
now = timezone.now()
categories = {}
herds = {}
maintainers = {}
wlog = None
if not nolog:
wlog = WorldLog()
wlog.datetime = now
for cat in Package.objects.values('category').distinct():
clog = CategoryLog()
clog.datetime = now
clog.category = cat['category']
categories[clog.category] = clog
for herd in Herd.objects.all():
hlog = HerdLog()
hlog.datetime = now
hlog.herd = herd
herds[herd.id] = hlog
for maintainer in Maintainer.objects.all():
mlog = MaintainerLog()
mlog.datetime = now
mlog.maintainer = maintainer
maintainers[maintainer.id] = mlog
package_queryset = Package.objects.all()
n_versions = {}
n_packaged = {}
n_overlay = {}
last_versions_gentoo = {}
last_versions_overlay = {}
last_versions_upstream = {}
if not fast:
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
for version in Version.objects.all().values(*attrs):
overlay, packaged = version['overlay'], version['packaged']
package_id = version['package_id']
_add_safe(n_versions, package_id)
if not packaged:
_add_last_ver(last_versions_upstream, version)
continue
if overlay == 'gentoo':
_add_safe(n_packaged, package_id)
_add_last_ver(last_versions_gentoo, version)
else:
_add_safe(n_overlay, package_id)
_add_last_ver(last_versions_overlay, version)
for package in package_queryset.select_related('herds', 'maintainers'):
if not fast:
package.n_versions = n_versions.get(package.id, 0)
package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0)
default = {'id': None}
package.last_version_gentoo_id = last_versions_gentoo.get(
package.id, default
)['id']
package.last_version_overlay_id = last_versions_overlay.get(
package.id, default
)['id']
package.last_version_upstream_id = last_versions_upstream.get(
package.id, default
)['id']
package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged \
+ package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay \
< package.n_versions)
def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
def update_log(storage, qs):
for row in qs:
update_row(storage, row['id'])
if not nolog:
update_log(herds, package.herds.all().values('id'))
update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category)
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
if nolog:
return
for clog in categories.values():
logger.info('+ [cl] %s', clog)
charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save()
for hlog in herds.values():
logger.info('+ [hl] %s', hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save()
for mlog in maintainers.values():
logger.info('+ [ml] %s', mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()
charts.rrd_update('world', now, wlog)
wlog.save()
def update_counters(fast=False, nolog=False, logger=None):
logger = logger or FakeLogger()
now = timezone.now()
categories = {}
herds = {}
maintainers = {}
wlog = None
if not nolog:
wlog = WorldLog()
wlog.datetime = now
for cat in Package.objects.values('category').distinct():
clog = CategoryLog()
clog.datetime = now
clog.category = cat['category']
categories[clog.category] = clog
for herd in Herd.objects.all():
hlog = HerdLog()
hlog.datetime = now
hlog.herd = herd
herds[herd.id] = hlog
for maintainer in Maintainer.objects.all():
mlog = MaintainerLog()
mlog.datetime = now
mlog.maintainer = maintainer
maintainers[maintainer.id] = mlog
package_queryset = Package.objects.all()
n_versions = {}
n_packaged = {}
n_overlay = {}
last_versions_gentoo = {}
last_versions_overlay = {}
last_versions_upstream = {}
if not fast:
attrs = ['id', 'version', 'overlay', 'packaged', 'package_id']
for version in Version.objects.all().values(*attrs):
overlay, packaged = version['overlay'], version['packaged']
package_id = version['package_id']
_add_safe(n_versions, package_id)
if not packaged:
_add_last_ver(last_versions_upstream, version)
continue
if overlay == 'gentoo':
_add_safe(n_packaged, package_id)
_add_last_ver(last_versions_gentoo, version)
else:
_add_safe(n_overlay, package_id)
_add_last_ver(last_versions_overlay, version)
for package in package_queryset.select_related('herds', 'maintainers'):
if not fast:
package.n_versions = n_versions.get(package.id, 0)
package.n_packaged = n_packaged.get(package.id, 0)
package.n_overlay = n_overlay.get(package.id, 0)
default = {'id': None}
package.last_version_gentoo_id = last_versions_gentoo.get(
package.id, default)['id']
package.last_version_overlay_id = last_versions_overlay.get(
package.id, default)['id']
package.last_version_upstream_id = last_versions_upstream.get(
package.id, default)['id']
package.save()
n_packages_gentoo = int(package.n_packaged == package.n_versions)
n_packages_overlay = int(package.n_overlay and package.n_packaged \
+ package.n_overlay == package.n_versions)
n_packages_outdated = int(package.n_packaged + package.n_overlay \
< package.n_versions)
def update_row(storage, key):
storage[key].n_packages_gentoo += n_packages_gentoo
storage[key].n_packages_overlay += n_packages_overlay
storage[key].n_packages_outdated += n_packages_outdated
storage[key].n_versions_gentoo += package.n_packaged
storage[key].n_versions_overlay += package.n_overlay
storage[key].n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
def update_log(storage, qs):
for row in qs:
update_row(storage, row['id'])
if not nolog:
update_log(herds, package.herds.all().values('id'))
update_log(maintainers, package.maintainers.all().values('id'))
update_row(categories, package.category)
wlog.n_packages_gentoo += n_packages_gentoo
wlog.n_packages_overlay += n_packages_overlay
wlog.n_packages_outdated += n_packages_outdated
wlog.n_versions_gentoo += package.n_packaged
wlog.n_versions_overlay += package.n_overlay
wlog.n_versions_upstream += package.n_versions - \
package.n_packaged - \
package.n_overlay
if nolog:
return
for clog in categories.values():
logger.info('+ [cl] %s', clog)
charts.rrd_update('category-%s' % clog.category, now, clog)
clog.save()
for hlog in herds.values():
logger.info('+ [hl] %s', hlog)
charts.rrd_update('herd-%d' % hlog.herd.id, now, hlog)
hlog.save()
for mlog in maintainers.values():
logger.info('+ [ml] %s', mlog)
charts.rrd_update('maintainer-%d' % mlog.maintainer.id, now, mlog)
mlog.save()
charts.rrd_update('world', now, wlog)
wlog.save()