def calculate_lag_stats(generated=None):
"""Return statistics about the scan times of all repositories."""
projects = get_projects()
stats = []
now = generated or datetime.now()
for project in projects:
repos = get_active_repos(project.name)
lag_stats = models.ProjectLagStats(
project=project.name,
generated=now,
total_active_repos=len(repos),
unscanned_repos=len([r for r in repos if not r.last_scanned]),
scanned_repos=len([r for r in repos if r.last_scanned]),
repos_without_root=len(
[r for r in repos if not r.root_commit_scanned]),
repos_with_root=len([r for r in repos if r.root_commit_scanned]))
scan_lag_with_repo = {}
scan_lag = []
for repo in repos:
if repo.last_scanned:
repo_scan_lag = (now - repo.last_scanned).total_seconds()
scan_lag.append(repo_scan_lag)
scan_lag_with_repo[repo_scan_lag] = '%s:%s' % (project.name,
repo.repo)
if scan_lag:
lag_stats.p50 = numpy.percentile(scan_lag, 50)
lag_stats.p75 = numpy.percentile(scan_lag, 75)
lag_stats.p90 = numpy.percentile(scan_lag, 90)
lag_stats.p95 = numpy.percentile(scan_lag, 95)
lag_stats.p99 = numpy.percentile(scan_lag, 99)
lag_stats.max = numpy.max(scan_lag)
lag_stats.min = numpy.min(scan_lag)
lag_stats.most_lagging_repo = scan_lag_with_repo[lag_stats.max]
stats.append(lag_stats)
return models.ProjectLagList(projects=stats, generated=now)
def test_empty_calculate_lag_stats(self):
my_project = model_helpers.create_project()
my_project.put()
generated_time = datetime.datetime(1970, 01, 02)
expected = models.ProjectLagList(
generated=generated_time,
projects=[
models.ProjectLagStats(
project=my_project.name,
total_active_repos=0,
repos_without_root=0,
repos_with_root=0,
scanned_repos=0,
unscanned_repos=0,
generated=generated_time,
),
],
)
generated = controller.calculate_lag_stats(generated=generated_time)
self.assertEqual(expected, generated)
def test_calculate_lag_stats(self):
model_helpers.create_project().put()
my_repo = model_helpers.create_repo()
my_repo.put()
second_repo = model_helpers.create_repo()
second_repo.repo = 'cooler'
second_repo.root_commit_scanned = True
second_repo.last_scanned = None
second_repo.put()
generated_time = datetime.datetime(1970, 01, 02)
expected = models.ProjectLagList(
generated=generated_time,
projects=[
models.ProjectLagStats(project=my_repo.project,
total_active_repos=2,
repos_without_root=1,
repos_with_root=1,
scanned_repos=1,
unscanned_repos=1,
generated=generated_time,
most_lagging_repo='%s:%s' %
(my_repo.project, my_repo.repo),
max=float(24 * 60 * 60),
min=float(24 * 60 * 60),
p50=float(24 * 60 * 60),
p75=float(24 * 60 * 60),
p90=float(24 * 60 * 60),
p95=float(24 * 60 * 60),
p99=float(24 * 60 * 60)),
],
)
generated = controller.calculate_lag_stats(generated=generated_time)
self.assertEqual(expected, generated)