def run(args):

    header = ['User', 'Tasks', 'Pushes', 'Tasks / Push']
    if args.sort_key < 0 or len(header) - 1 < args.sort_key:
        RequestParser.error("invalid value for 'sort_key'")

    args.branches = 'try'
    limit = args.limit
    delattr(args, 'limit')
    pushes = run_query('user_pushes', args)
    pushes = pushes['data']

    tasks = run_query('user_tasks', args)['data']

    users = defaultdict(list)
    for user, num in tasks:
        users[user].append(num)
    for user, num in pushes:
        users[user].append(num)

    data = []
    for user, value in users.items():
        if len(value) != 2:
            continue
        tasks, pushes = value
        data.append([user, tasks, pushes, round(float(tasks) / pushes, 2)])

    data = sorted(data, key=lambda k: k[args.sort_key], reverse=True)
    data = data[:limit]
    data.insert(0, header)
    return data
Example #2
0
def run(config, context):
    push_tests = run_query('unique_tests', config, context)['data']
    push_tests = {item[0]: item[1] for item in push_tests if item[0] is not None}

    context.pushid = None
    all_tests = run_query('unique_tests', config, context)['data']
    all_tests = {item[0]: item[1] for item in all_tests if item[0] is not None}

    data = []
    total_push = 0
    total_num = 0
    for platform, num in all_tests.items():
        push_num = push_tests.get(platform, 0)
        percentage = round(float(push_num)/num * 100, 1)
        data.append([platform, percentage])

        total_num += num
        total_push += push_num

    data.sort()

    total_percentage = round(float(total_push)/total_num * 100, 1)
    data.append(['total', total_percentage])
    data.insert(0, ['Platform', 'Percentage of Tests Run'])
    return data
Example #3
0
def run(args):

    pushes = len(set(run_query('all_push_id', args)['data']['push.id']))
    backouts = len(set(run_query('backout_rate', args)['data']['push.id']))
    backout_rate = round((float(backouts) / pushes) * 100, 2)

    return (
        ['Pushes', 'Backouts', 'Backout Rate'],
        [pushes, backouts, backout_rate],
    )
Example #4
0
def run(args):

    pushes = len(set(run_query('all_push_id', args)['data']['push.id']))
    backouts = len(set(run_query('backout_rate', args)['data']['push.id']))
    backout_rate = round((float(backouts) / pushes) * 100, 2)

    args.branches = ['try']
    data = run_query('total_hours_spent_on_branch', args)['data']

    try_hours = int(data['hours'])
    try_efficiency = round(10000000 / (backout_rate * try_hours), 2)

    return (
        ['Backout Rate', 'Total Compute Hours on Try', 'Try Efficiency'],
        [backout_rate, try_hours, try_efficiency],
    )
Example #5
0
def test_missing_manifests():
    """
    Ensure all suites (except a blacklist) are generating manifest information.
    """
    BLACKLIST = (
        "web-platform-tests",
        "talos",
        "web-platform-tests-reftests",
        "web-platform-tests-wdspec",
        "web-platform-tests-crashtests",
        "jittest",
        "geckoview-junit",
        "cppunittest",
        "test-verify-wpt",
        None,
    )
    ALLOWED_MISSING = 5

    result = run_query("test_missing_manifests", Namespace())

    for suite, count in result["data"]:
        if suite not in BLACKLIST:
            assert count < ALLOWED_MISSING, f"{suite} is missing manifest information"

    # Ensure the blacklist doesn't contain more than necessary.
    found_suites = {suite for suite, count in result["data"]}
    for suite in BLACKLIST:
        assert suite in found_suites, f"{suite} might be removed from the blacklist"
Example #6
0
def test_missing_result_manifests():
    """
    Ensure unittest results from all manifest-based suites (except an ignorelist)
    have information on what manifest the result corresponds to.
    """
    IGNORELIST = {
        "marionette",
    }
    ALLOWED_MISSING = 70

    result = run_query("test_missing_result_manifests", Namespace())

    missing = []

    for suite, count in result["data"]:
        if suite not in IGNORELIST:
            if count > ALLOWED_MISSING:
                missing.append((suite, count))

    assert missing == []

    # Ensure the ignorelist doesn't contain more than necessary.
    unignorable = []
    found_suites = {suite: count for suite, count in result["data"]}
    for suite in IGNORELIST:
        if suite not in found_suites or found_suites[suite] < ALLOWED_MISSING:
            unignorable.append(suite)

    assert unignorable == []
Example #7
0
def run(args):

    result = run_query('skipped_tests', args)['data']

    result.sort(key=lambda x: x[0])
    result.insert(0, ['Result test', 'run suite', 'count'])
    return result
Example #8
0
def make_push_objects(**kwargs):
    result = run_query("push_revisions", Namespace(**kwargs))

    pushes = []

    for pushid, date, revs, parents in result["data"]:
        topmost = list(set(revs) - set(parents))[0]

        cur = Push([topmost] + [r for r in revs if r != topmost])

        # avoids the need to query hgmo to find this info
        cur._id = pushid
        cur._date = date

        pushes.append(cur)

    pushes.sort(key=lambda p: p._id)

    for i, cur in enumerate(pushes):
        if i != 0:
            cur._parent = pushes[i - 1]

        if i != len(pushes) - 1:
            cur._child = pushes[i + 1]

    return pushes
Example #9
0
    def _decision_artifact_urls(self):
        """All artifact urls from the Decision task of this push.

        Returns:
            list: A list of urls.
        """
        return run_query('decision_artifacts', Namespace(rev=self.rev))['data']
Example #10
0
def run(args):

    if not args.table:
        data = run_query('meta', args)['data']
        data = sorted([(d['name'], ) for d in data])
        data.insert(0, ('Table', ))
        return data

    if not args.attribute:
        data = run_query('meta_columns', args)['data']
        data = sorted([(d['name'], ) for d in data])
        data.insert(0, ('Column', ))
        return data

    data = run_query('meta_values', args)['data']
    data.insert(0, (args.attribute, 'count'))
    return data
Example #11
0
def run(args):
    """
    THIS IS PRONE TO DOUBLE COUNTING, AS DIFFERENT TEST CHUNKS COVER COMMON LINES
    AT THE VERY LEAST YOU GET A ROUGH ESTIMATE OF COVERAGE
    """

    result = run_query('code_coverage', args)
    output = [result['header']] + result['data']
    return output
def run(args):
    result = []
    data = run_query('tests_by_suite', args)['data']
    data = [row for row in data if row[0] is not None]

    for suite, _, num_tests in sorted(data):
        result.append([suite, num_tests])
    result.insert(0, ['Suite', 'Number of tests'])
    return result
Example #13
0
def run(args):

    data = run_query('try_commit_messages', args)['data']

    # Order is important as the search stops after the first successful test.
    d = OrderedDict()
    d.update(subcommand('syntax'))
    d['vanilla'] = {
        'test': 'try:',
        'method': 'vanilla try syntax',
    }
    d.update(subcommand('fuzzy'))
    d.update(subcommand('again'))
    d.update(subcommand('empty'))
    d.update(subcommand('release'))
    d.update(subcommand('coverage'))
    d.update(subcommand('chooser'))
    d['other'] = {
        'test': '',
        'method': 'other',
    }
    d['total'] = {
        'test': None,
        'method': 'total',
    }

    data = zip(data['user'], data['message'])

    count = defaultdict(int)
    users = defaultdict(set)

    for user, message in data:
        if user == 'reviewbot':
            continue

        for k, v in d.items():
            if v['test'] in message:
                count[k] += 1
                users[k].add(user)
                break

    count['total'] = sum(count.values())
    users['total'] = set(chain(*users.values()))

    def fmt(key):
        percent = round(float(count[key]) / count['total'] * 100, 1)
        return [
            d[key]['method'], count[key], percent,
            len(users[key]),
            round(float(count[key]) / len(users[key]), 2)
        ]  # noqa

    data = [['Method', 'Pushes', 'Percent', 'Users', 'Push / User']]
    for k, v in sorted(count.items(), key=lambda t: t[1], reverse=True):
        data.append(fmt(k))
    return data
def run(args):

    header = [
        'Revision', 'Files With Coverage', 'Total Files',
        'Percent with Coverage'
    ]
    covered_files = run_query('covered_files', args)['data']

    if None in [item for items in covered_files for item in items]:
        raise MissingDataError("ActiveData returned null value.")

    total_files = run_query('total_files', args)['data']

    if None in [item for items in total_files for item in items]:
        raise MissingDataError("ActiveData returned null value.")

    by_revision = {}
    by_date = {}
    for item in covered_files:
        # if we don't have 100 artifacts, something is broken, no data, or still ingesting
        if item[2] >= 100:
            # default total files=-1, in some cases this is reported
            by_revision[item[0]] = {'covered': item[3], 'total': -1}
            by_date[item[1]] = item[0]

    for item in total_files:
        if item[0] in by_revision:
            by_revision[item[0]]['total'] = item[2]

    data = []
    dates = sorted(by_date.keys(), reverse=True)

    for date in dates[0:args.limit]:
        rev = by_date[date]
        covered = by_revision[rev]['covered']
        total = by_revision[rev]['total']
        if covered < 0 or total < 0:
            continue
        data.append(
            [rev, covered, total,
             round((float(covered) / total) * 100, 1)])
    data.insert(0, header)
    return data
Example #15
0
def run(args):
    logger.info("Running the 'build_times' recipe!")

    # The task_durations query was designed to be more general purpose than
    # this recipe. Since we are only looking at build tasks here, we can hard
    # code the 'kind' context.
    args.kind = "build"

    # Set the default branch if it wasn't specified.
    args.branches = args.branches or DEFAULT_BRANCHES

    # The 'run_query' function will run the specified query (named in the
    # 'queries' directory) using the specified context. Often you can just
    # forward 'args' wholesale, but you can also build the context manually if
    # necessary. In more complicated cases the output of one query might
    # determine the context of another.
    data = run_query('task_durations', args)['data']

    # The above query returns a list of values of the form:
    # [<build label>, <number of tasks>, <average runtime>]
    result = []
    for record in data:
        # ActiveData can sometimes return missing records or erroneous values.
        # Sometimes data sanitization is needed.
        if record[2] is None:
            continue

        # Compute the total hours spent running each build by multiplying
        # number of tasks by average runtime.
        record.append(int(round(record[1] * record[2], 0)))
        result.append(record)

        # Round the average hours.
        record[2] = round(record[2])

        # In this case, we don't care about the number of tasks after
        # calculating the total.
        del record[1]

    # Sort the results by average runtime.
    result = sorted(result, key=lambda k: k[1], reverse=True)

    # Insert a header.
    result.insert(0, ['Build', 'Average (min)', 'Total (min)'])

    # Return the result to be formatted into a table by adr. In this case the
    # structure of result looks like:
    # [
    #   [ "HeaderA", "HeaderB", "HeaderC"],
    #   [ "Row1 Col1", "Row1 Col2", "Row1 Col3"],
    #   [ "Row2 Col1", "Row2 Col2", "Row2 Col3"],
    #   etc..
    # ]
    return result
def get_stats_for_week(args):
    # query all jobs that are fixed by commit- build a map and determine for each regression:
    # <fixed_rev>: [{<broken_rev>: "time_from_build_to_job", "job_name">}, ...]
    backouts = run_query('fixed_by_commit_jobs', args)['data']
    if backouts == {}:
        return []
    builddate = backouts['build.date']
    jobname = backouts['job.type.name']
    jobdate = backouts['action.request_time']
    buildrev = backouts['build.revision12']
    fixedrev = backouts['failure.notes.text']
    fbc = {}
    if len(builddate) != len(fixedrev) != len(buildrev) != len(jobname) != len(
            jobdate):
        print("invalid length detected in the data found")

    counter = -1
    for item in fixedrev:
        counter += 1
        if counter > len(fixedrev):
            break
        if item is None:
            continue

        # sometimes we have a list and some items are None
        if isinstance(item, list):
            i = None
            iter = 0
            while i is None and iter < len(item):
                i = item[iter]
                iter += 1
            if i is None:
                continue
            item = i

        item = item[0:12]
        if item not in fbc:
            fbc[item] = {}

        if buildrev[counter] not in fbc[item]:
            fbc[item][buildrev[counter]] = False
        # 300 seconds is a magic number, represents lag between
        # build finished and test job scheduled
        if (jobdate[counter] - builddate[counter]) < 300:
            fbc[item][buildrev[counter]] = True

    results = []
    for item in fbc:
        failed = [x for x in fbc[item] if not fbc[item][x]]
        passfail = "pass"
        if len(failed):
            passfail = "fail"
        results.append([item, passfail])
    return results
Example #17
0
def run(args):

    result = []

    data = run_query('test_durations', args)['data']['result.test']

    duration = [1, 2, 5, 10, 20, 30, 45, 60, 90, 120, 150, 'max']
    for index in range(0, len(duration)):
        result.append([duration[index], data[index]])
    result.insert(0, ['Max Duration (seconds)', 'number of tests'])
    return result
Example #18
0
def run(args):

    if args.test_name == '':
        args.test_name = '(~(file.*|http.*))'
        args.platform_config = "test-%s/%s" % (args.platform, args.build_type)
    else:
        args.test_name = '.*%s.*' % args.test_name
        args.platform_config = "test-"
        args.groupby = 'run.key'
        args.result = ["T", "F"]

    result = run_query('intermittent_tests', args)['data']
    total_runs = run_query('intermittent_test_rate', args)['data']

    intermittent_tests = []
    for item in result['run.key']:
        parts = item.split('/')
        config = "%s/%s" % (parts[0], parts[1].split('-')[0])
        if config not in intermittent_tests:
            intermittent_tests.append(config)

    retVal = {}
    for test in total_runs:
        parts = test[0].split('/')
        config = "%s/%s" % (parts[0], parts[1].split('-')[0])
        if config in intermittent_tests:
            if config not in retVal:
                retVal[config] = [test[1], test[2]]
            else:
                retVal[config][0] += test[1]
                retVal[config][1] += test[2]

    result = []
    for item in retVal:
        val = [item]
        val.extend(retVal[item])
        result.append(val)
    result.insert(0, ['Config', 'Failures', 'Runs'])
    return result
Example #19
0
def test_good_result_manifests():
    """
    Ensure there are no bad manifest paths in recent result manifest information.
    """
    result = run_query("test_all_result_groups", Namespace())

    for group, count in result["data"]:
        if group is None:
            continue

        assert (
            not task.is_bad_group("x", group) and "\\" not in group
        ), f"{group} group is bad!"
Example #20
0
def run(args):
    # These 4 args are defined so that we can share the queries with the
    # 'intermittent_test_data' recipe.
    args.test_name = '(~(file.*|http.*))'
    args.groupby = 'result.test'
    args.result = ["F"]
    args.platform_config = "test-%s/%s" % (args.platform, args.build_type)

    jobs = run_query('intermittent_jobs', args)['data']
    result = run_query('intermittent_tests', args)['data']
    total_runs = run_query('intermittent_test_rate', args)['data']

    intermittent_tests = []
    # for each result, match up the revision/name with jobs, if a match, save testname
    index = -1
    for item in result['result.test']:
        index += 1
        rev = result['repo.changeset.id12'][index]
        jobname = result['run.key'][index]
        if rev not in jobs['repo.changeset.id12']:
            continue

        index = jobs['repo.changeset.id12'].index(rev)

        if jobname != jobs['job.type.name'][index]:
            continue

        intermittent_tests.append(item)

    result = []
    for test in total_runs:
        if test[1] == 0:
            continue

        if test[0] in intermittent_tests:
            result.append(test)

    result.insert(0, ['Testname', 'Failures', 'Runs'])
    return result
Example #21
0
def test_good_result_manifests():
    """
    Ensure there are no bad manifest paths in recent result manifest information.
    """
    result = run_query("test_all_result_groups", Namespace())

    for group, label, _ in result["data"]:
        if group is None:
            continue

        if any(s in label for s in {"web-platform-tests", "test-verify-wpt"}):
            group = task.wpt_workaround(group)

        assert (not task.is_bad_group("x", group) and "\\"
                not in group), f"{group} group for task {label} is bad!"
Example #22
0
def test_good_manifests():
    """
    Ensure there are no bad manifest paths in recent manifest information.
    """
    result = run_query("test_all_groups", Namespace())

    for (groups,) in result["data"]:
        if groups is None:
            continue

        if not isinstance(groups, list):
            groups = [groups]

        for group in groups:
            assert (
                not task.is_bad_group("x", group) and "\\" not in group
            ), f"{group} group is bad!"
Example #23
0
def make_push_objects(**kwargs):
    data = run_query("push_revisions", Namespace(**kwargs))["data"]

    pushes = []
    cur = prev = None
    for pushid, revs, parents in data:
        topmost = list(set(revs) - set(parents))[0]

        cur = Push(topmost)
        if prev:
            # avoids the need to query hgmo to find parent pushes
            cur._parent = prev

        pushes.append(cur)
        prev = cur

    return pushes
def run(args):

    limit = args.limit
    delattr(args, 'limit')

    data = run_query('task_durations', args)['data']
    result = []
    for record in data:
        if record[2] is None:
            continue
        record[2] = round(record[2] / 60, 2)
        record.append(int(round(record[1] * record[2], 0)))
        result.append(record)

    result = sorted(result, key=lambda k: k[args.sort_key],
                    reverse=True)[:limit]
    result.insert(0, ['Taskname', 'Num Jobs', 'Average Hours', 'Total Hours'])
    return result
def run(args):

    result = run_query("perf_tp6_compare", args)

    tests, revisions = result["edges"]
    header = (["Test", "Subtest"] +
              [p["name"]
               for p in revisions["domain"]["partitions"]] + ["Change"])
    suites, tests = zip(*(p["value"] for p in tests["domain"]["partitions"]))
    values = result["data"]["result.stats.median"]

    data = list([s, scrub_suite(s, t)] +
                [round(v) if v is not None else None
                 for v in row] + [change(*row)]
                for s, t, row in zip(suites, tests, values))

    data.insert(0, header)
    return data
Example #26
0
def test_missing_result_manifests():
    """
    Ensure unittest results from all manifest-based suites (except a blacklist)
    have information on what manifest the result corresponds to.
    """
    BLACKLIST = {"marionette"}
    ALLOWED_MISSING = 70

    result = run_query("test_missing_result_manifests", Namespace())

    for suite, count in result["data"]:
        if suite not in BLACKLIST:
            assert (
                count < ALLOWED_MISSING
            ), f"{suite} is missing result manifest information"

    # Ensure the blacklist doesn't contain more than necessary.
    found_suites = {suite for suite, count in result["data"]}
    for suite in BLACKLIST:
        assert suite in found_suites, f"{suite} might be removed from the blacklist"
def run(args):
    # process config data
    data = run_query('config_intermittents', args)["data"]
    result = []
    for record in data:
        if not record or not record[args.sort_key]:
            continue
        if isinstance(record[1], list):
            record[1] = record[1][-1]
        if record[2] is None:
            continue
        if record[3] is None:
            continue
        record.append(float(record[3] / (record[2] * 1.0)))
        result.append(record)

    result = sorted(result, key=lambda k: k[args.sort_key],
                    reverse=True)[:args.limit]
    result.insert(
        0, ['Platform', 'Type', 'Num Jobs', 'Number failed', '%% failed'])
    return result
Example #28
0
def run(config, args):
    # process config data
    data = run_query('config_durations', config, args)["data"]
    result = []
    for record in data:
        if not record or not record[args.sort_key]:
            continue
        if isinstance(record[1], list):
            record[1] = record[1][-1]
        if record[2] is None:
            continue
        if record[3] is None:
            continue
        record[3] = round(record[3] / 60, 2)
        record.append(int(round(record[2] * record[3], 0)))
        result.append(record)

    result = sorted(result, key=lambda k: k[args.sort_key],
                    reverse=True)[:args.limit]
    result.insert(
        0, ['Platform', 'Type', 'Num Jobs', 'Average Hours', 'Total Hours'])
    return result
def run(args):
    results = []
    branches = args.branches
    delattr(args, 'branches')

    total = 0
    for branch in branches:
        args.branches = [branch]
        data = run_query('total_hours_spent_on_branch', args)['data']
        hours = int(data['hours'])
        total += hours
        results.append([branch, hours])

    results.append(["total", total])

    for res in results:
        percentage = round(float(res[1]) / total * 100, 1)
        res.append(percentage)

    results.sort(key=lambda x: x[1], reverse=True)
    results.insert(0, ['Branch', 'Total Compute Hours', 'Percentage'])
    return results
Example #30
0
    def tasks(self):
        """All tasks that ran on the push, including retriggers and backfills.

        Returns:
            list: A list of `Task` objects.
        """
        args = Namespace(rev=self.rev)
        data = run_query('push_results', args)['data']

        tasks = []
        for kwargs in data:
            # Do a bit of data sanitization.
            if any(a not in kwargs
                   for a in ('label', 'duration', 'result', 'classification')):
                continue

            if kwargs['duration'] <= 0:
                continue

            tasks.append(Task(**kwargs))

        return tasks