def run(args):
header = ['User', 'Tasks', 'Pushes', 'Tasks / Push']
if args.sort_key < 0 or len(header) - 1 < args.sort_key:
RequestParser.error("invalid value for 'sort_key'")
args.branches = 'try'
limit = args.limit
delattr(args, 'limit')
pushes = run_query('user_pushes', args)
pushes = pushes['data']
tasks = run_query('user_tasks', args)['data']
users = defaultdict(list)
for user, num in tasks:
users[user].append(num)
for user, num in pushes:
users[user].append(num)
data = []
for user, value in users.items():
if len(value) != 2:
continue
tasks, pushes = value
data.append([user, tasks, pushes, round(float(tasks) / pushes, 2)])
data = sorted(data, key=lambda k: k[args.sort_key], reverse=True)
data = data[:limit]
data.insert(0, header)
return data
def run(config, context):
push_tests = run_query('unique_tests', config, context)['data']
push_tests = {item[0]: item[1] for item in push_tests if item[0] is not None}
context.pushid = None
all_tests = run_query('unique_tests', config, context)['data']
all_tests = {item[0]: item[1] for item in all_tests if item[0] is not None}
data = []
total_push = 0
total_num = 0
for platform, num in all_tests.items():
push_num = push_tests.get(platform, 0)
percentage = round(float(push_num)/num * 100, 1)
data.append([platform, percentage])
total_num += num
total_push += push_num
data.sort()
total_percentage = round(float(total_push)/total_num * 100, 1)
data.append(['total', total_percentage])
data.insert(0, ['Platform', 'Percentage of Tests Run'])
return data
def run(args):
pushes = len(set(run_query('all_push_id', args)['data']['push.id']))
backouts = len(set(run_query('backout_rate', args)['data']['push.id']))
backout_rate = round((float(backouts) / pushes) * 100, 2)
return (
['Pushes', 'Backouts', 'Backout Rate'],
[pushes, backouts, backout_rate],
)
def run(args):
pushes = len(set(run_query('all_push_id', args)['data']['push.id']))
backouts = len(set(run_query('backout_rate', args)['data']['push.id']))
backout_rate = round((float(backouts) / pushes) * 100, 2)
args.branches = ['try']
data = run_query('total_hours_spent_on_branch', args)['data']
try_hours = int(data['hours'])
try_efficiency = round(10000000 / (backout_rate * try_hours), 2)
return (
['Backout Rate', 'Total Compute Hours on Try', 'Try Efficiency'],
[backout_rate, try_hours, try_efficiency],
)
def test_missing_manifests():
"""
Ensure all suites (except a blacklist) are generating manifest information.
"""
BLACKLIST = (
"web-platform-tests",
"talos",
"web-platform-tests-reftests",
"web-platform-tests-wdspec",
"web-platform-tests-crashtests",
"jittest",
"geckoview-junit",
"cppunittest",
"test-verify-wpt",
None,
)
ALLOWED_MISSING = 5
result = run_query("test_missing_manifests", Namespace())
for suite, count in result["data"]:
if suite not in BLACKLIST:
assert count < ALLOWED_MISSING, f"{suite} is missing manifest information"
# Ensure the blacklist doesn't contain more than necessary.
found_suites = {suite for suite, count in result["data"]}
for suite in BLACKLIST:
assert suite in found_suites, f"{suite} might be removed from the blacklist"
def test_missing_result_manifests():
"""
Ensure unittest results from all manifest-based suites (except an ignorelist)
have information on what manifest the result corresponds to.
"""
IGNORELIST = {
"marionette",
}
ALLOWED_MISSING = 70
result = run_query("test_missing_result_manifests", Namespace())
missing = []
for suite, count in result["data"]:
if suite not in IGNORELIST:
if count > ALLOWED_MISSING:
missing.append((suite, count))
assert missing == []
# Ensure the ignorelist doesn't contain more than necessary.
unignorable = []
found_suites = {suite: count for suite, count in result["data"]}
for suite in IGNORELIST:
if suite not in found_suites or found_suites[suite] < ALLOWED_MISSING:
unignorable.append(suite)
assert unignorable == []
def run(args):
result = run_query('skipped_tests', args)['data']
result.sort(key=lambda x: x[0])
result.insert(0, ['Result test', 'run suite', 'count'])
return result
def make_push_objects(**kwargs):
result = run_query("push_revisions", Namespace(**kwargs))
pushes = []
for pushid, date, revs, parents in result["data"]:
topmost = list(set(revs) - set(parents))[0]
cur = Push([topmost] + [r for r in revs if r != topmost])
# avoids the need to query hgmo to find this info
cur._id = pushid
cur._date = date
pushes.append(cur)
pushes.sort(key=lambda p: p._id)
for i, cur in enumerate(pushes):
if i != 0:
cur._parent = pushes[i - 1]
if i != len(pushes) - 1:
cur._child = pushes[i + 1]
return pushes
def _decision_artifact_urls(self):
"""All artifact urls from the Decision task of this push.
Returns:
list: A list of urls.
"""
return run_query('decision_artifacts', Namespace(rev=self.rev))['data']
def run(args):
if not args.table:
data = run_query('meta', args)['data']
data = sorted([(d['name'], ) for d in data])
data.insert(0, ('Table', ))
return data
if not args.attribute:
data = run_query('meta_columns', args)['data']
data = sorted([(d['name'], ) for d in data])
data.insert(0, ('Column', ))
return data
data = run_query('meta_values', args)['data']
data.insert(0, (args.attribute, 'count'))
return data
def run(args):
"""
THIS IS PRONE TO DOUBLE COUNTING, AS DIFFERENT TEST CHUNKS COVER COMMON LINES
AT THE VERY LEAST YOU GET A ROUGH ESTIMATE OF COVERAGE
"""
result = run_query('code_coverage', args)
output = [result['header']] + result['data']
return output
def run(args):
result = []
data = run_query('tests_by_suite', args)['data']
data = [row for row in data if row[0] is not None]
for suite, _, num_tests in sorted(data):
result.append([suite, num_tests])
result.insert(0, ['Suite', 'Number of tests'])
return result
def run(args):
data = run_query('try_commit_messages', args)['data']
# Order is important as the search stops after the first successful test.
d = OrderedDict()
d.update(subcommand('syntax'))
d['vanilla'] = {
'test': 'try:',
'method': 'vanilla try syntax',
}
d.update(subcommand('fuzzy'))
d.update(subcommand('again'))
d.update(subcommand('empty'))
d.update(subcommand('release'))
d.update(subcommand('coverage'))
d.update(subcommand('chooser'))
d['other'] = {
'test': '',
'method': 'other',
}
d['total'] = {
'test': None,
'method': 'total',
}
data = zip(data['user'], data['message'])
count = defaultdict(int)
users = defaultdict(set)
for user, message in data:
if user == 'reviewbot':
continue
for k, v in d.items():
if v['test'] in message:
count[k] += 1
users[k].add(user)
break
count['total'] = sum(count.values())
users['total'] = set(chain(*users.values()))
def fmt(key):
percent = round(float(count[key]) / count['total'] * 100, 1)
return [
d[key]['method'], count[key], percent,
len(users[key]),
round(float(count[key]) / len(users[key]), 2)
] # noqa
data = [['Method', 'Pushes', 'Percent', 'Users', 'Push / User']]
for k, v in sorted(count.items(), key=lambda t: t[1], reverse=True):
data.append(fmt(k))
return data
def run(args):
header = [
'Revision', 'Files With Coverage', 'Total Files',
'Percent with Coverage'
]
covered_files = run_query('covered_files', args)['data']
if None in [item for items in covered_files for item in items]:
raise MissingDataError("ActiveData returned null value.")
total_files = run_query('total_files', args)['data']
if None in [item for items in total_files for item in items]:
raise MissingDataError("ActiveData returned null value.")
by_revision = {}
by_date = {}
for item in covered_files:
# if we don't have 100 artifacts, something is broken, no data, or still ingesting
if item[2] >= 100:
# default total files=-1, in some cases this is reported
by_revision[item[0]] = {'covered': item[3], 'total': -1}
by_date[item[1]] = item[0]
for item in total_files:
if item[0] in by_revision:
by_revision[item[0]]['total'] = item[2]
data = []
dates = sorted(by_date.keys(), reverse=True)
for date in dates[0:args.limit]:
rev = by_date[date]
covered = by_revision[rev]['covered']
total = by_revision[rev]['total']
if covered < 0 or total < 0:
continue
data.append(
[rev, covered, total,
round((float(covered) / total) * 100, 1)])
data.insert(0, header)
return data
def run(args):
logger.info("Running the 'build_times' recipe!")
# The task_durations query was designed to be more general purpose than
# this recipe. Since we are only looking at build tasks here, we can hard
# code the 'kind' context.
args.kind = "build"
# Set the default branch if it wasn't specified.
args.branches = args.branches or DEFAULT_BRANCHES
# The 'run_query' function will run the specified query (named in the
# 'queries' directory) using the specified context. Often you can just
# forward 'args' wholesale, but you can also build the context manually if
# necessary. In more complicated cases the output of one query might
# determine the context of another.
data = run_query('task_durations', args)['data']
# The above query returns a list of values of the form:
# [<build label>, <number of tasks>, <average runtime>]
result = []
for record in data:
# ActiveData can sometimes return missing records or erroneous values.
# Sometimes data sanitization is needed.
if record[2] is None:
continue
# Compute the total hours spent running each build by multiplying
# number of tasks by average runtime.
record.append(int(round(record[1] * record[2], 0)))
result.append(record)
# Round the average hours.
record[2] = round(record[2])
# In this case, we don't care about the number of tasks after
# calculating the total.
del record[1]
# Sort the results by average runtime.
result = sorted(result, key=lambda k: k[1], reverse=True)
# Insert a header.
result.insert(0, ['Build', 'Average (min)', 'Total (min)'])
# Return the result to be formatted into a table by adr. In this case the
# structure of result looks like:
# [
# [ "HeaderA", "HeaderB", "HeaderC"],
# [ "Row1 Col1", "Row1 Col2", "Row1 Col3"],
# [ "Row2 Col1", "Row2 Col2", "Row2 Col3"],
# etc..
# ]
return result
def get_stats_for_week(args):
# query all jobs that are fixed by commit- build a map and determine for each regression:
# <fixed_rev>: [{<broken_rev>: "time_from_build_to_job", "job_name">}, ...]
backouts = run_query('fixed_by_commit_jobs', args)['data']
if backouts == {}:
return []
builddate = backouts['build.date']
jobname = backouts['job.type.name']
jobdate = backouts['action.request_time']
buildrev = backouts['build.revision12']
fixedrev = backouts['failure.notes.text']
fbc = {}
if len(builddate) != len(fixedrev) != len(buildrev) != len(jobname) != len(
jobdate):
print("invalid length detected in the data found")
counter = -1
for item in fixedrev:
counter += 1
if counter > len(fixedrev):
break
if item is None:
continue
# sometimes we have a list and some items are None
if isinstance(item, list):
i = None
iter = 0
while i is None and iter < len(item):
i = item[iter]
iter += 1
if i is None:
continue
item = i
item = item[0:12]
if item not in fbc:
fbc[item] = {}
if buildrev[counter] not in fbc[item]:
fbc[item][buildrev[counter]] = False
# 300 seconds is a magic number, represents lag between
# build finished and test job scheduled
if (jobdate[counter] - builddate[counter]) < 300:
fbc[item][buildrev[counter]] = True
results = []
for item in fbc:
failed = [x for x in fbc[item] if not fbc[item][x]]
passfail = "pass"
if len(failed):
passfail = "fail"
results.append([item, passfail])
return results
def run(args):
result = []
data = run_query('test_durations', args)['data']['result.test']
duration = [1, 2, 5, 10, 20, 30, 45, 60, 90, 120, 150, 'max']
for index in range(0, len(duration)):
result.append([duration[index], data[index]])
result.insert(0, ['Max Duration (seconds)', 'number of tests'])
return result
def run(args):
if args.test_name == '':
args.test_name = '(~(file.*|http.*))'
args.platform_config = "test-%s/%s" % (args.platform, args.build_type)
else:
args.test_name = '.*%s.*' % args.test_name
args.platform_config = "test-"
args.groupby = 'run.key'
args.result = ["T", "F"]
result = run_query('intermittent_tests', args)['data']
total_runs = run_query('intermittent_test_rate', args)['data']
intermittent_tests = []
for item in result['run.key']:
parts = item.split('/')
config = "%s/%s" % (parts[0], parts[1].split('-')[0])
if config not in intermittent_tests:
intermittent_tests.append(config)
retVal = {}
for test in total_runs:
parts = test[0].split('/')
config = "%s/%s" % (parts[0], parts[1].split('-')[0])
if config in intermittent_tests:
if config not in retVal:
retVal[config] = [test[1], test[2]]
else:
retVal[config][0] += test[1]
retVal[config][1] += test[2]
result = []
for item in retVal:
val = [item]
val.extend(retVal[item])
result.append(val)
result.insert(0, ['Config', 'Failures', 'Runs'])
return result
def test_good_result_manifests():
"""
Ensure there are no bad manifest paths in recent result manifest information.
"""
result = run_query("test_all_result_groups", Namespace())
for group, count in result["data"]:
if group is None:
continue
assert (
not task.is_bad_group("x", group) and "\\" not in group
), f"{group} group is bad!"
def run(args):
# These 4 args are defined so that we can share the queries with the
# 'intermittent_test_data' recipe.
args.test_name = '(~(file.*|http.*))'
args.groupby = 'result.test'
args.result = ["F"]
args.platform_config = "test-%s/%s" % (args.platform, args.build_type)
jobs = run_query('intermittent_jobs', args)['data']
result = run_query('intermittent_tests', args)['data']
total_runs = run_query('intermittent_test_rate', args)['data']
intermittent_tests = []
# for each result, match up the revision/name with jobs, if a match, save testname
index = -1
for item in result['result.test']:
index += 1
rev = result['repo.changeset.id12'][index]
jobname = result['run.key'][index]
if rev not in jobs['repo.changeset.id12']:
continue
index = jobs['repo.changeset.id12'].index(rev)
if jobname != jobs['job.type.name'][index]:
continue
intermittent_tests.append(item)
result = []
for test in total_runs:
if test[1] == 0:
continue
if test[0] in intermittent_tests:
result.append(test)
result.insert(0, ['Testname', 'Failures', 'Runs'])
return result
def test_good_result_manifests():
"""
Ensure there are no bad manifest paths in recent result manifest information.
"""
result = run_query("test_all_result_groups", Namespace())
for group, label, _ in result["data"]:
if group is None:
continue
if any(s in label for s in {"web-platform-tests", "test-verify-wpt"}):
group = task.wpt_workaround(group)
assert (not task.is_bad_group("x", group) and "\\"
not in group), f"{group} group for task {label} is bad!"
def test_good_manifests():
"""
Ensure there are no bad manifest paths in recent manifest information.
"""
result = run_query("test_all_groups", Namespace())
for (groups,) in result["data"]:
if groups is None:
continue
if not isinstance(groups, list):
groups = [groups]
for group in groups:
assert (
not task.is_bad_group("x", group) and "\\" not in group
), f"{group} group is bad!"
def make_push_objects(**kwargs):
data = run_query("push_revisions", Namespace(**kwargs))["data"]
pushes = []
cur = prev = None
for pushid, revs, parents in data:
topmost = list(set(revs) - set(parents))[0]
cur = Push(topmost)
if prev:
# avoids the need to query hgmo to find parent pushes
cur._parent = prev
pushes.append(cur)
prev = cur
return pushes
def run(args):
limit = args.limit
delattr(args, 'limit')
data = run_query('task_durations', args)['data']
result = []
for record in data:
if record[2] is None:
continue
record[2] = round(record[2] / 60, 2)
record.append(int(round(record[1] * record[2], 0)))
result.append(record)
result = sorted(result, key=lambda k: k[args.sort_key],
reverse=True)[:limit]
result.insert(0, ['Taskname', 'Num Jobs', 'Average Hours', 'Total Hours'])
return result
def run(args):
result = run_query("perf_tp6_compare", args)
tests, revisions = result["edges"]
header = (["Test", "Subtest"] +
[p["name"]
for p in revisions["domain"]["partitions"]] + ["Change"])
suites, tests = zip(*(p["value"] for p in tests["domain"]["partitions"]))
values = result["data"]["result.stats.median"]
data = list([s, scrub_suite(s, t)] +
[round(v) if v is not None else None
for v in row] + [change(*row)]
for s, t, row in zip(suites, tests, values))
data.insert(0, header)
return data
def test_missing_result_manifests():
"""
Ensure unittest results from all manifest-based suites (except a blacklist)
have information on what manifest the result corresponds to.
"""
BLACKLIST = {"marionette"}
ALLOWED_MISSING = 70
result = run_query("test_missing_result_manifests", Namespace())
for suite, count in result["data"]:
if suite not in BLACKLIST:
assert (
count < ALLOWED_MISSING
), f"{suite} is missing result manifest information"
# Ensure the blacklist doesn't contain more than necessary.
found_suites = {suite for suite, count in result["data"]}
for suite in BLACKLIST:
assert suite in found_suites, f"{suite} might be removed from the blacklist"
def run(args):
# process config data
data = run_query('config_intermittents', args)["data"]
result = []
for record in data:
if not record or not record[args.sort_key]:
continue
if isinstance(record[1], list):
record[1] = record[1][-1]
if record[2] is None:
continue
if record[3] is None:
continue
record.append(float(record[3] / (record[2] * 1.0)))
result.append(record)
result = sorted(result, key=lambda k: k[args.sort_key],
reverse=True)[:args.limit]
result.insert(
0, ['Platform', 'Type', 'Num Jobs', 'Number failed', '%% failed'])
return result
def run(config, args):
# process config data
data = run_query('config_durations', config, args)["data"]
result = []
for record in data:
if not record or not record[args.sort_key]:
continue
if isinstance(record[1], list):
record[1] = record[1][-1]
if record[2] is None:
continue
if record[3] is None:
continue
record[3] = round(record[3] / 60, 2)
record.append(int(round(record[2] * record[3], 0)))
result.append(record)
result = sorted(result, key=lambda k: k[args.sort_key],
reverse=True)[:args.limit]
result.insert(
0, ['Platform', 'Type', 'Num Jobs', 'Average Hours', 'Total Hours'])
return result
def run(args):
results = []
branches = args.branches
delattr(args, 'branches')
total = 0
for branch in branches:
args.branches = [branch]
data = run_query('total_hours_spent_on_branch', args)['data']
hours = int(data['hours'])
total += hours
results.append([branch, hours])
results.append(["total", total])
for res in results:
percentage = round(float(res[1]) / total * 100, 1)
res.append(percentage)
results.sort(key=lambda x: x[1], reverse=True)
results.insert(0, ['Branch', 'Total Compute Hours', 'Percentage'])
return results
def tasks(self):
"""All tasks that ran on the push, including retriggers and backfills.
Returns:
list: A list of `Task` objects.
"""
args = Namespace(rev=self.rev)
data = run_query('push_results', args)['data']
tasks = []
for kwargs in data:
# Do a bit of data sanitization.
if any(a not in kwargs
for a in ('label', 'duration', 'result', 'classification')):
continue
if kwargs['duration'] <= 0:
continue
tasks.append(Task(**kwargs))
return tasks