Beispiel #1
0
def aggregate_flaky_tests(day=None, max_flaky_tests=200):
    if day is None:
        day = datetime.utcnow().date() - timedelta(days=1)

    try:
        projects = Project.query.all()

        for project in projects:
            tests = get_flaky_tests(day, day + timedelta(days=1), [project], max_flaky_tests)

            for test in tests:
                _log_metrics(
                    "flaky_test_reruns",
                    flaky_test_reruns_name=test['name'],
                    flaky_test_reruns_project_id=test['project_id'],
                    flaky_test_reruns_flaky_runs=test['flaky_runs'],
                    flaky_test_reruns_passing_runs=test['passing_runs'],
                )
                try_create(FlakyTestStat, {
                    'name': test['name'],
                    'project_id': test['project_id'],
                    'date': day,
                    'last_flaky_run_id': test['id'],
                    'flaky_runs': test['flaky_runs'],
                    'double_reruns': test['double_reruns'],
                    'passing_runs': test['passing_runs'],
                })
                # Potentially hundreds of commits per project may be a bit excessive,
                # but the metric posting can potentially take seconds, meaning this could be
                # a very long-running transaction otherwise.
                db.session.commit()

        db.session.commit()
    except Exception as err:
        logging.exception(unicode(err))
Beispiel #2
0
def aggregate_flaky_tests(day=None, max_flaky_tests=200):
    if day is None:
        day = datetime.utcnow().date() - timedelta(days=1)

    try:
        projects = Project.query.all()

        for project in projects:
            tests = get_flaky_tests(day, day + timedelta(days=1), [project], max_flaky_tests)

            for test in tests:
                first_run = (
                    db.session.query(TestCase.date_created)
                    .filter(TestCase.project_id == test["project_id"], TestCase.name_sha == test["hash"])
                    .order_by(TestCase.date_created)
                    .limit(1)
                    .scalar()
                )

                log_metrics(
                    "flaky_test_reruns",
                    flaky_test_reruns_name=test["name"],
                    flaky_test_reruns_project_id=test["project_id"],
                    flaky_test_reruns_flaky_runs=test["flaky_runs"],
                    flaky_test_reruns_passing_runs=test["passing_runs"],
                )
                try_create(
                    FlakyTestStat,
                    {
                        "name": test["name"],
                        "project_id": test["project_id"],
                        "date": day,
                        "last_flaky_run_id": test["id"],
                        "flaky_runs": test["flaky_runs"],
                        "double_reruns": test["double_reruns"],
                        "passing_runs": test["passing_runs"],
                        "first_run": first_run,
                    },
                )

        db.session.commit()
    except Exception as err:
        logging.exception(unicode(err))
Beispiel #3
0
def aggregate_flaky_tests(day=None, max_flaky_tests=200):
    if day is None:
        day = datetime.utcnow().date() - timedelta(days=1)

    try:
        projects = Project.query.all()

        for project in projects:
            tests = get_flaky_tests(day, day + timedelta(days=1), [project], max_flaky_tests)

            for test in tests:
                first_run = db.session.query(
                    TestCase.date_created
                ).filter(
                    TestCase.project_id == test['project_id'],
                    TestCase.name_sha == test['hash']
                ).order_by(
                    TestCase.date_created
                ).limit(1).scalar()

                log_metrics(
                    "flaky_test_reruns",
                    flaky_test_reruns_name=test['name'],
                    flaky_test_reruns_project_id=test['project_id'],
                    flaky_test_reruns_flaky_runs=test['flaky_runs'],
                    flaky_test_reruns_passing_runs=test['passing_runs'],
                )
                try_create(FlakyTestStat, {
                    'name': test['name'],
                    'project_id': test['project_id'],
                    'date': day,
                    'last_flaky_run_id': test['id'],
                    'flaky_runs': test['flaky_runs'],
                    'double_reruns': test['double_reruns'],
                    'passing_runs': test['passing_runs'],
                    'first_run': first_run
                })

        db.session.commit()
    except Exception as err:
        logging.exception(unicode(err))
Beispiel #4
0
def aggregate_flaky_tests(day=None, max_flaky_tests=200):
    if day is None:
        day = datetime.utcnow().date() - timedelta(days=1)

    try:
        projects = Project.query.all()

        for project in projects:
            tests = get_flaky_tests(day, day + timedelta(days=1), [project],
                                    max_flaky_tests)

            for test in tests:
                first_run = db.session.query(TestCase.date_created).filter(
                    TestCase.project_id == test['project_id'],
                    TestCase.name_sha == test['hash']).order_by(
                        TestCase.date_created).limit(1).scalar()

                log_metrics(
                    "flaky_test_reruns",
                    flaky_test_reruns_name=test['name'],
                    flaky_test_reruns_project_id=test['project_id'],
                    flaky_test_reruns_flaky_runs=test['flaky_runs'],
                    flaky_test_reruns_passing_runs=test['passing_runs'],
                )
                try_create(
                    FlakyTestStat, {
                        'name': test['name'],
                        'project_id': test['project_id'],
                        'date': day,
                        'last_flaky_run_id': test['id'],
                        'flaky_runs': test['flaky_runs'],
                        'double_reruns': test['double_reruns'],
                        'passing_runs': test['passing_runs'],
                        'first_run': first_run
                    })

        db.session.commit()
    except Exception as err:
        logging.exception(unicode(err))
Beispiel #5
0
    def generate(self, days=7):
        end_period = datetime.utcnow()
        days_delta = timedelta(days=days)
        start_period = end_period - days_delta

        # if we're pulling data for a select number of days let's use the
        # previous week as the previous period
        if days < 7:
            previous_end_period = end_period - timedelta(days=7)
        else:
            previous_end_period = start_period
        previous_start_period = previous_end_period - days_delta

        current_results = self.get_project_stats(
            start_period, end_period)
        previous_results = self.get_project_stats(
            previous_start_period, previous_end_period)

        for project, stats in current_results.items():
            # exclude projects that had no builds in this period
            if not stats['total_builds']:
                del current_results[project]
                continue

            previous_stats = previous_results.get(project)
            if not previous_stats:
                green_change = None
                duration_change = None
            elif stats['green_percent'] is None:
                green_change = None
                duration_change = None
            elif previous_stats['green_percent'] is None:
                green_change = None
                duration_change = None
            else:
                green_change = stats['green_percent'] - previous_stats['green_percent']
                duration_change = stats['avg_duration'] - previous_stats['avg_duration']

            if not previous_stats:
                total_change = None
            elif previous_stats['total_builds'] is None:
                total_change = None
            else:
                total_change = stats['total_builds'] - previous_stats['total_builds']

            stats['avg_duration'] = stats['avg_duration']

            stats['total_change'] = total_change
            stats['percent_change'] = green_change
            stats['duration_change'] = duration_change

        project_stats = sorted(
            current_results.items(), key=lambda x: (
                -(x[1]['total_builds'] or 0), abs(x[1]['green_percent'] or 0),
                x[0].name,
            ))

        current_failure_stats = self.get_failure_stats(
            start_period, end_period)
        previous_failure_stats = self.get_failure_stats(
            previous_start_period, previous_end_period)
        failure_stats = []
        for stat_name, current_stat_value in current_failure_stats['reasons'].iteritems():
            previous_stat_value = previous_failure_stats['reasons'].get(stat_name, 0)
            failure_stats.append({
                'name': stat_name,
                'current': {
                    'value': current_stat_value,
                    'percent': percent(current_stat_value, current_failure_stats['total'])
                },
                'previous': {
                    'value': previous_stat_value,
                    'percent': percent(previous_stat_value, previous_failure_stats['total'])
                },
            })

        flaky_tests = get_flaky_tests(
            start_period, end_period, self.projects, MAX_FLAKY_TESTS)
        slow_tests = self.get_slow_tests(start_period, end_period)

        title = 'Build Report ({0} through {1})'.format(
            start_period.strftime('%b %d, %Y'),
            end_period.strftime('%b %d, %Y'),
        )
        if len(self.projects) == 1:
            title = '[%s] %s' % (iter(self.projects).next().name, title)

        return {
            'title': title,
            'period': [start_period, end_period],
            'failure_stats': failure_stats,
            'project_stats': project_stats,
            'tests': {
                'flaky_list': flaky_tests,
                'slow_list': slow_tests,
            },
        }
Beispiel #6
0
    def generate(self, days=7):
        end_period = datetime.utcnow()
        days_delta = timedelta(days=days)
        start_period = end_period - days_delta

        # if we're pulling data for a select number of days let's use the
        # previous week as the previous period
        if days < 7:
            previous_end_period = end_period - timedelta(days=7)
        else:
            previous_end_period = start_period
        previous_start_period = previous_end_period - days_delta

        current_results = self.get_project_stats(start_period, end_period)
        previous_results = self.get_project_stats(previous_start_period,
                                                  previous_end_period)

        for project, stats in current_results.items():
            # exclude projects that had no builds in this period
            if not stats['total_builds']:
                del current_results[project]
                continue

            previous_stats = previous_results.get(project)
            if not previous_stats:
                green_change = None
                duration_change = None
            elif stats['green_percent'] is None:
                green_change = None
                duration_change = None
            elif previous_stats['green_percent'] is None:
                green_change = None
                duration_change = None
            else:
                green_change = stats['green_percent'] - previous_stats[
                    'green_percent']
                duration_change = stats['avg_duration'] - previous_stats[
                    'avg_duration']

            if not previous_stats:
                total_change = None
            elif previous_stats['total_builds'] is None:
                total_change = None
            else:
                total_change = stats['total_builds'] - previous_stats[
                    'total_builds']

            stats['avg_duration'] = stats['avg_duration']

            stats['total_change'] = total_change
            stats['percent_change'] = green_change
            stats['duration_change'] = duration_change

        project_stats = sorted(current_results.items(),
                               key=lambda x: (
                                   -(x[1]['total_builds'] or 0),
                                   abs(x[1]['green_percent'] or 0),
                                   x[0].name,
                               ))

        current_failure_stats = self.get_failure_stats(start_period,
                                                       end_period)
        previous_failure_stats = self.get_failure_stats(
            previous_start_period, previous_end_period)
        failure_stats = []
        for stat_name, current_stat_value in current_failure_stats[
                'reasons'].iteritems():
            previous_stat_value = previous_failure_stats['reasons'].get(
                stat_name, 0)
            failure_stats.append({
                'name': stat_name,
                'current': {
                    'value':
                    current_stat_value,
                    'percent':
                    percent(current_stat_value, current_failure_stats['total'])
                },
                'previous': {
                    'value':
                    previous_stat_value,
                    'percent':
                    percent(previous_stat_value,
                            previous_failure_stats['total'])
                },
            })

        flaky_tests = get_flaky_tests(start_period, end_period, self.projects,
                                      MAX_FLAKY_TESTS)
        slow_tests = self.get_slow_tests(start_period, end_period)

        title = 'Build Report ({0} through {1})'.format(
            start_period.strftime('%b %d, %Y'),
            end_period.strftime('%b %d, %Y'),
        )
        if len(self.projects) == 1:
            title = '[%s] %s' % (iter(self.projects).next().name, title)

        return {
            'title': title,
            'period': [start_period, end_period],
            'failure_stats': failure_stats,
            'project_stats': project_stats,
            'tests': {
                'flaky_list': flaky_tests,
                'slow_list': slow_tests,
            },
        }