def test_update_build_summary(self): values1 = [1, 2, 3, 4, 5] values2 = [2, 4, 6, 8] new_test_run = self.build1.test_runs.create(environment=self.env1) new_test_run.metrics.create(name='new_foo', suite=self.suite1, result=5) self.receive_testrun(self.build1.version, self.env1.slug, tests_file='{"suite1/new_foo": "pass"}') summary1 = BuildSummary.create_or_update(self.build1, self.env1) summary2 = BuildSummary.create_or_update(self.build1, self.env2) self.assertTrue(summary1.has_metrics) self.assertTrue(eq(geomean(values1), summary1.metrics_summary)) self.assertTrue(summary2.has_metrics) self.assertTrue(eq(geomean(values2), summary2.metrics_summary)) self.assertEqual(5, summary1.tests_total) self.assertEqual(2, summary1.tests_pass) self.assertEqual(1, summary1.tests_fail) self.assertEqual(1, summary1.tests_skip) self.assertEqual(1, summary1.tests_xfail) self.assertEqual(3, summary2.tests_total) self.assertEqual(1, summary2.tests_pass) self.assertEqual(1, summary2.tests_fail) self.assertEqual(1, summary2.tests_skip) self.assertEqual(0, summary2.tests_xfail)
def test_environment_summary(self): values1 = [1, 2, 3, 4] values2 = [2, 4, 6, 8] summary1 = MetricsSummary(self.build1, self.env1) summary2 = MetricsSummary(self.build1, self.env2) self.assertTrue(summary1.has_metrics) self.assertTrue(summary2.has_metrics) self.assertTrue(eq(geomean(values1), summary1.value)) self.assertTrue(eq(geomean(values2), summary2.value))
def __call__(testrun): if testrun.status_recorded: return status = defaultdict(lambda: Status(test_run=testrun)) for test in testrun.tests.all(): sid = test.suite_id if test.result is True: status[None].tests_pass = status[None].tests_pass + 1 status[sid].tests_pass = status[sid].tests_pass + 1 elif test.result is False: status[None].tests_fail = status[None].tests_fail + 1 status[sid].tests_fail = status[sid].tests_fail + 1 else: status[None].tests_skip = status[None].tests_skip + 1 status[sid].tests_skip = status[sid].tests_skip + 1 metrics = defaultdict(lambda: []) for metric in testrun.metrics.all(): sid = metric.suite_id for v in metric.measurement_list: metrics[None].append(v) metrics[sid].append(v) for sid, values in metrics.items(): status[sid].metrics_summary = geomean(values) for sid, s in status.items(): s.suite_id = sid s.save() testrun.status_recorded = True testrun.save()
def create_or_update_projectstatus(apps, schema_editor): ProjectStatus = apps.get_model('core', 'ProjectStatus') Build = apps.get_model('core', 'Build') Metric = apps.get_model('core', 'Metric') for build in Build.objects.order_by('datetime').prefetch_related('status'): try: status = build.status except ProjectStatus.DoesNotExist: previous = ProjectStatus.objects.filter( build__project=build.project, build__datetime__lt=build.datetime, ).last() status = ProjectStatus(build=build, previous=previous) metrics = Metric.objects.filter(test_run__build_id=build.id).all() metrics_summary = geomean([m.result for m in metrics]) test_summary = TestSummary(build) status.tests_pass = test_summary.tests_pass status.tests_fail = test_summary.tests_fail status.tests_skip = test_summary.tests_skip status.metrics_summary = metrics_summary status.last_updated = timezone.now() status.save()
def __call__(testrun): if testrun.status_recorded: return status = defaultdict(lambda: Status(test_run=testrun)) # Get number of passing tests per suite passes = testrun.tests.filter(result=True).values('suite_id').annotate( pass_count=Count('suite_id')).order_by() xfails = testrun.tests.filter( result=False, has_known_issues=True).values('suite_id').annotate( xfail_count=Count('suite_id')).order_by() fails = testrun.tests.filter(result=False).exclude( has_known_issues=True).values('suite_id').annotate( fail_count=Count('suite_id')).order_by() skips = testrun.tests.filter( result__isnull=True).values('suite_id').annotate( skip_count=Count('suite_id')).order_by() for p in passes: status[None].tests_pass += p['pass_count'] status[p['suite_id']].tests_pass += p['pass_count'] for x in xfails: status[None].tests_xfail += x['xfail_count'] status[x['suite_id']].tests_xfail += x['xfail_count'] for f in fails: status[None].tests_fail += f['fail_count'] status[f['suite_id']].tests_fail += f['fail_count'] for s in skips: status[None].tests_skip += s['skip_count'] status[s['suite_id']].tests_skip += s['skip_count'] metrics = defaultdict(lambda: []) for metric in testrun.metrics.all(): sid = metric.suite_id for v in metric.measurement_list: metrics[None].append(v) metrics[sid].append(v) # One Status has many test suites and each of one of them # has their own summary (i.e. geomean). # The status having no test suite (suite=None) represent # the TestRun's summary if len(metrics[None]): status[None].has_metrics = True for sid, values in metrics.items(): status[sid].metrics_summary = geomean(values) status[sid].has_metrics = True for sid, s in status.items(): s.suite_id = sid s.suite_version = get_suite_version(testrun, s.suite) s.save() testrun.status_recorded = True testrun.save()
def __init__(self, build, environment=None): queryset = Metric.objects.filter(test_run__build_id=build.id) if environment: queryset = queryset.filter(test_run__environment_id=environment.id) metrics = queryset.all() values = [m.result for m in metrics] self.value = geomean(values) self.has_metrics = len(values) > 0
def test_basic_build_summary(self): values1 = [1, 2, 3, 4] values2 = [2, 4, 6, 8] summary1 = BuildSummary.create_or_update(self.build1, self.env1) summary2 = BuildSummary.create_or_update(self.build1, self.env2) self.assertTrue(summary1.has_metrics) self.assertTrue(summary2.has_metrics) self.assertTrue(eq(geomean(values1), summary1.metrics_summary)) self.assertTrue(eq(geomean(values2), summary2.metrics_summary)) self.assertEqual(4, summary1.tests_total) self.assertEqual(1, summary1.tests_pass) self.assertEqual(1, summary1.tests_fail) self.assertEqual(1, summary1.tests_skip) self.assertEqual(1, summary1.tests_xfail) self.assertEqual(3, summary2.tests_total) self.assertEqual(1, summary2.tests_pass) self.assertEqual(1, summary2.tests_fail) self.assertEqual(1, summary2.tests_skip) self.assertEqual(0, summary2.tests_xfail)
def __call__(testrun): if testrun.status_recorded: return status = defaultdict(lambda: Status(test_run=testrun)) for test in testrun.tests.all(): sid = test.suite_id if test.result is True: status[None].tests_pass = status[None].tests_pass + 1 status[sid].tests_pass = status[sid].tests_pass + 1 elif test.result is False: status[None].tests_fail = status[None].tests_fail + 1 status[sid].tests_fail = status[sid].tests_fail + 1 else: status[None].tests_skip = status[None].tests_skip + 1 status[sid].tests_skip = status[sid].tests_skip + 1 metrics = defaultdict(lambda: []) for metric in testrun.metrics.all(): sid = metric.suite_id for v in metric.measurement_list: metrics[None].append(v) metrics[sid].append(v) for sid, values in metrics.items(): status[sid].metrics_summary = geomean(values) for sid, s in status.items(): s.suite_id = sid s.save() testrun.status_recorded = True testrun.save() status = ProjectStatus.create_or_update(testrun.build) if status.finished: try: notify_project_status.delay(status.id) except OSError as e: # can't request background task for some reason; log the error # and continue. # # This will happen as "OSError: [Errno 111] Connection refused" # in development environments without a running AMQP server, # but also on production setups that are not running the # background job processes because they don't need email # notifications or CI integration logger.error("Cannot schedule notification: " + str(e) + "\n" + traceback.format_exc())
def get_dynamic_summary(project, environments, metrics, date_start, date_end): entry = {} filters = [] if not metrics: for env in environments: entry[env] = [] return entry for m in metrics: suite, metric = parse_name(m) filters.append(Q(suite__slug=suite) & Q(name=metric)) metric_filter = reduce(lambda x, y: x | y, filters) data = models.Metric.objects.filter( test_run__build__project=project, test_run__environment__slug__in=environments, test_run__created_at__range=(date_start, date_end), ).filter(metric_filter).prefetch_related( 'test_run', 'test_run__environment', 'test_run__build', 'test_run__build__annotation', ).order_by('test_run__environment__id', 'test_run__build__id') for environment, metrics_by_environment in groupby( data, lambda m: m.test_run.environment): envdata = [] metrics_by_build = groupby(metrics_by_environment, lambda m: m.test_run.build) for build, metric_list in metrics_by_build: values = [] for metric in metric_list: if not metric.is_outlier: values = values + metric.measurement_list try: description = build.annotation.description except ObjectDoesNotExist: description = "" envdata.append([ build.datetime.timestamp(), geomean(values), build.version, description, ]) entry[environment.slug] = sorted(envdata, key=(lambda e: e[0])) return entry
def __call__(testrun): if testrun.status_recorded: return status = defaultdict(lambda: Status(test_run=testrun)) for test in testrun.tests.all(): sid = test.suite_id if test.result is True: status[None].tests_pass += 1 status[sid].tests_pass += 1 elif test.result is False: if test.known_issues.exists(): status[None].tests_xfail += 1 status[sid].tests_xfail += 1 else: status[None].tests_fail += 1 status[sid].tests_fail += 1 else: status[None].tests_skip += 1 status[sid].tests_skip += 1 metrics = defaultdict(lambda: []) for metric in testrun.metrics.all(): sid = metric.suite_id for v in metric.measurement_list: metrics[None].append(v) metrics[sid].append(v) # One Status has many test suites and each of one of them # has their own summary (i.e. geomean). # The status having no test suite (suite=None) represent # the TestRun's summary if len(metrics[None]): status[None].has_metrics = True for sid, values in metrics.items(): status[sid].metrics_summary = geomean(values) status[sid].has_metrics = True for sid, s in status.items(): s.suite_id = sid s.suite_version = get_suite_version(testrun, s.suite) s.save() testrun.status_recorded = True testrun.save()
def __call__(testrun): if testrun.status_recorded: return status = defaultdict(lambda: Status(test_run=testrun)) for test in testrun.tests.all(): sid = test.suite_id if test.result is True: status[None].tests_pass += 1 status[sid].tests_pass += 1 elif test.result is False: if test.known_issues.exists(): status[None].tests_xfail += 1 status[sid].tests_xfail += 1 else: status[None].tests_fail += 1 status[sid].tests_fail += 1 else: status[None].tests_skip += 1 status[sid].tests_skip += 1 metrics = defaultdict(lambda: []) for metric in testrun.metrics.all(): sid = metric.suite_id for v in metric.measurement_list: metrics[None].append(v) metrics[sid].append(v) for sid, values in metrics.items(): status[sid].metrics_summary = geomean(values) status[sid].has_metrics = True for sid, s in status.items(): s.suite_id = sid s.suite_version = get_suite_version(testrun, s.suite) s.save() testrun.status_recorded = True testrun.save()
def test_empty_set(self): self.assertAlmostEqual(0, geomean([]))
def __init__(self, build): metrics = Metric.objects.filter(test_run__build_id=build.id).all() values = [m.result for m in metrics] self.value = geomean(values) self.has_metrics = len(values) > 0
def test_set_with_only_invalid_values(self): self.assertAlmostEqual(0, geomean([0]))
def test_basic_summary(self): values = [1, 2, 3, 4, 2, 4, 6, 8] summary = MetricsSummary(self.build1) self.assertTrue(summary.has_metrics) self.assertTrue(eq(geomean(values), summary.value))
def test_basic(self): self.assertAlmostEqual(3.1622, geomean([1, 10]), 3)
def test_exclude_negative_numbers(self): self.assertAlmostEqual(4, geomean([4, -1, 4]))
def test_exclude_zeroes(self): self.assertAlmostEqual(4, geomean([4, 0, 4]))