def analyze_variants_all_tests_1_platform(self, platform, vary):
# generate one graph image for results of all benchmarks
# on one platform and one kernel, comparing effects of
# two or more combos of kernel options (test run attributes)
# (numa_fake,stale_page,kswapd_merge,sched_idle, etc)
kernel = self.cgiform.getvalue('kernel', 'some_kernel')
self.passthru.append('kernel=%s' % kernel)
# two or more vary_groups, one for each plotted line,
# each group begins with vary= and ends with next &
# each group has comma-separated list of test attribute key=val pairs
# eg vary=keyval1,keyval2&vary=keyval3,keyval4
vary_groups = [
dict(pair.split('=', 1) for pair in vary_group.split(','))
for vary_group in vary
]
test = self.benchmarks[0] # pick any test in all jobs
kernels, test_tag = self.jobs_selector(test, self.job_table,
self.kernel_dates)
linekeys = {}
plot_data = {}
baselines = {}
for i, vary_group in enumerate(vary_groups):
group_attributes = self.test_attributes.copy()
group_attributes.update(vary_group)
linekey = ','.join('%s=%s' % (attr, vary_group[attr])
for attr in vary_group)
linekeys[i] = linekey
data = {}
for benchmark in self.benchmarks:
metric = perf.benchmark_main_metric(benchmark)
runs = collect_testruns(self.job_table[kernel],
benchmark + test_tag, group_attributes,
self.platforms_filter, 'by_hosts'
in self.toggles, self.no_antag)
vals = []
for testrunx in runs[platform]:
vals += perf.get_metric_at_point([testrunx], metric)
if vals:
if benchmark not in baselines:
baselines[benchmark], stddev = plotgraph.avg_dev(vals)
vals = [val / baselines[benchmark] for val in vals]
data[benchmark] = vals
plot_data[i] = data
title = "%s on %s" % (kernel, platform)
for attr in self.test_attributes:
title += ', %s=%s' % (attr, self.test_attributes[attr])
if 'table' in self.cgiform:
self.table_for_variants_all_tests(title,
plot_data,
linekeys,
range(len(linekeys)),
filtered_passthru=self.passthru,
test_tag=test_tag)
else:
graph_variants_all_tests(title, plot_data, linekeys, self.size,
'dark' in self.toggles)
def table_testrun_details(runs, metric, tko_server, show_attrs):
print "<table border=1 cellpadding=4 cellspacing=0>"
print "<tr><td> %s metric </td>" % metric
print "<td> Job label </td> <td> Job tag </td> <td> Run results </td>"
print "<td> Started_time </td>"
if show_attrs:
print "<td> Test attributes </td>"
print "</tr>\n"
for testrunx in runs:
print "<tr> <td>",
vals = perf.get_metric_at_point([testrunx], metric)
for v in vals:
print "%.4g " % v,
print "</td>"
row = get_testrun_context(testrunx)
row[2] = ( "<a href='//%s/results/%s/%s/results/keyval'> %s </a>"
% (tko_server, row[1], row[2], row[2]) )
for v in row:
print "<td> %s </td>" % v
if show_attrs:
attrs = get_test_attributes(testrunx)
print "<td>",
for attr in sorted(attrs.keys()):
if attr == "sysinfo-cmdline": continue
if attr[:4] == "svs-": continue
val = attrs[attr]
if len(val) > 40:
val = val[:40-3] + "..."
print "%s=%s " % (attr, val)
print "</td>"
print "</tr>\n"
print "</table>"
def table_testrun_details(runs, metric, tko_server, show_attrs):
print "<table border=1 cellpadding=4 cellspacing=0>"
print "<tr><td> %s metric </td>" % metric
print "<td> Job label </td> <td> Job tag </td> <td> Run results </td>"
print "<td> Started_time </td>"
if show_attrs:
print "<td> Test attributes </td>"
print "</tr>\n"
for testrunx in runs:
print "<tr> <td>",
vals = perf.get_metric_at_point([testrunx], metric)
for v in vals:
print "%.4g " % v,
print "</td>"
row = get_testrun_context(testrunx)
row[2] = ("<a href='//%s/results/%s/%s/results/keyval'> %s </a>" %
(tko_server, row[1], row[2], row[2]))
for v in row:
print "<td> %s </td>" % v
if show_attrs:
attrs = get_test_attributes(testrunx)
print "<td>",
for attr in sorted(attrs.keys()):
if attr == "sysinfo-cmdline": continue
if attr[:4] == "svs-": continue
val = attrs[attr]
if len(val) > 40:
val = val[:40 - 3] + "..."
print "%s=%s " % (attr, val)
print "</td>"
print "</tr>\n"
print "</table>"
def analyze_variants_all_tests_1_platform(self, platform, vary):
# generate one graph image for results of all benchmarks
# on one platform and one kernel, comparing effects of
# two or more combos of kernel options (test run attributes)
# (numa_fake,stale_page,kswapd_merge,sched_idle, etc)
kernel = self.cgiform.getvalue('kernel', 'some_kernel')
self.passthru.append('kernel=%s' % kernel)
# two or more vary_groups, one for each plotted line,
# each group begins with vary= and ends with next &
# each group has comma-separated list of test attribute key=val pairs
# eg vary=keyval1,keyval2&vary=keyval3,keyval4
vary_groups = [dict(pair.split('=',1) for pair
in vary_group.split(','))
for vary_group in vary]
test = self.benchmarks[0] # pick any test in all jobs
kernels, test_tag = self.jobs_selector(test, self.job_table,
self.kernel_dates)
linekeys = {}
plot_data = {}
baselines = {}
for i, vary_group in enumerate(vary_groups):
group_attributes = self.test_attributes.copy()
group_attributes.update(vary_group)
linekey = ','.join('%s=%s' % (attr, vary_group[attr])
for attr in vary_group)
linekeys[i] = linekey
data = {}
for benchmark in self.benchmarks:
metric = perf.benchmark_main_metric(benchmark)
runs = collect_testruns(self.job_table[kernel],
benchmark+test_tag,
group_attributes,
self.platforms_filter,
'by_hosts' in self.toggles,
self.no_antag)
vals = []
for testrunx in runs[platform]:
vals += perf.get_metric_at_point([testrunx], metric)
if vals:
if benchmark not in baselines:
baselines[benchmark], stddev = plotgraph.avg_dev(vals)
vals = [val/baselines[benchmark] for val in vals]
data[benchmark] = vals
plot_data[i] = data
title = "%s on %s" % (kernel, platform)
for attr in self.test_attributes:
title += ', %s=%s' % (attr, self.test_attributes[attr])
if 'table' in self.cgiform:
self.table_for_variants_all_tests(title, plot_data, linekeys,
range(len(linekeys)),
filtered_passthru=self.passthru,
test_tag=test_tag)
else:
graph_variants_all_tests(title, plot_data, linekeys,
self.size, 'dark' in self.toggles)
def collect_raw_scores(runs, metric):
# get unscaled scores of test runs for 1 test on certain jobs
# arrange them by platform type
platform_scores = {} # platform --> list of perf scores
for platform in runs:
vals = perf.get_metric_at_point(runs[platform], metric)
if vals:
platform_scores[platform] = vals
return platform_scores
def collect_raw_scores(runs, metric):
# get unscaled scores of test runs for 1 test on certain jobs
# arrange them by platform type
platform_scores = {} # platform --> list of perf scores
for platform in runs:
vals = perf.get_metric_at_point(runs[platform], metric)
if vals:
platform_scores[platform] = vals
return platform_scores
def collect_scaled_scores(metric):
# get scores of test runs for 1 test on some kernels and platforms
# optionally make relative to first kernel on that platform
# arrange by plotline (ie platform) for gnuplot
plot_data = {} # platform --> (kernel --> list of perf scores)
baseline = {}
for kernel in sorted(test_runs.keys()):
for platform in test_runs[kernel]:
vals = perf.get_metric_at_point(test_runs[kernel][platform], metric)
if vals:
if relative:
if platform not in baseline:
baseline[platform], std = plotgraph.avg_dev(vals)
vals = [v/baseline[platform] for v in vals]
pdp = plot_data.setdefault(platform, {})
pdp.setdefault(kernel, []).extend(vals)
return plot_data
def collect_scaled_scores(metric, test_runs, regressed_platforms, relative):
# get scores of test runs for 1 test on some kernels and platforms
# optionally make relative to oldest (?) kernel on that platform
# arrange by plotline (ie platform) for gnuplot
plot_data = {} # platform --> (kernel --> list of perf scores)
baseline = {}
for kernel in sorted(test_runs.keys()):
for platform in test_runs[kernel]:
if not (regressed_platforms is None
or platform in regressed_platforms):
continue # delete results for uninteresting platforms
vals = perf.get_metric_at_point(test_runs[kernel][platform],
metric)
if vals:
if relative:
if platform not in baseline:
baseline[platform], std = plotgraph.avg_dev(vals)
vals = [v / baseline[platform] for v in vals]
pdp = plot_data.setdefault(platform, {})
pdp.setdefault(kernel, []).extend(vals)
return plot_data
def testrun_details_for_1_test_kernel_platform(test, metric, platform):
kernel = form.getvalue('kernel')
show_attrs = 'attrs' in form
print "Content-Type: text/html\n\n<html><body>"
print "<h3> %s %s: %s%s, Kernel %s on %s </h3>" % (
test_group, kernel_legend, test.capitalize(), suite_notes, kernel, platform)
print "<table border=1 cellpadding=4>"
print "<tr><td> Test_idx </td> <td> %s metric </td>" % metric
print "<td> Job tag </td> <td> Subdir </td> <td> Job label </td>"
print "<td> Started_time </td>"
if show_attrs:
print "<td> Test attributes </td>"
print "</tr>\n"
for testrunx in test_runs[kernel][platform]:
print "<tr> <td>", testrunx, "</td>"
print "<td>",
vals = perf.get_metric_at_point([testrunx], metric)
for v in vals:
print "%.4g" % v,
print "</td>"
row = get_testrun_context(testrunx)
print ( "<td> <a href='//%s/results/%s/%s/results'> %s </a></td>"
% (results_server, row[0], row[1], row[0]) )
for v in row[1:]:
print "<td> %s </td>" % v
if show_attrs:
attrs = perf.get_test_attributes(testrunx)
print "<td>",
for attr in attrs:
# if attr == "sysinfo-cmdline": continue
# if attr[:4] == "svs-": continue
val = attrs[attr]
if len(val) > 40:
val = val[:40-3] + "..."
print "%s=%s" % (attr, val)
print "</td>"
print "</tr>\n"
print "</table>"
print "</body></html>"
def collect_twoway_scores(metric, antagonists, twoway_runs, relative):
alone = ''
plot_data = {}
for kernel in twoway_runs:
for test2 in antagonists:
runs = twoway_runs[kernel].get(test2, [])
vals = perf.get_metric_at_point(runs, metric)
plot_data.setdefault(test2, {})
if vals:
plot_data[test2][kernel] = vals
if relative:
vals = plot_data[alone].get(kernel, [])
if vals:
baseline = perf.average(vals)
for test2 in antagonists:
vals = plot_data[test2].get(kernel, [])
vals = [val/baseline for val in vals]
if vals:
plot_data[test2][kernel] = vals
else:
for test2 in antagonists:
if kernel in plot_data[test2]:
del plot_data[test2][kernel]
return plot_data
def collect_twoway_scores(metric, antagonists, twoway_runs, relative):
alone = ''
plot_data = {}
for kernel in twoway_runs:
for test2 in antagonists:
runs = twoway_runs[kernel].get(test2, [])
vals = perf.get_metric_at_point(runs, metric)
plot_data.setdefault(test2, {})
if vals:
plot_data[test2][kernel] = vals
if relative:
vals = plot_data[alone].get(kernel, [])
if vals:
baseline = perf.average(vals)
for test2 in antagonists:
vals = plot_data[test2].get(kernel, [])
vals = [val / baseline for val in vals]
if vals:
plot_data[test2][kernel] = vals
else:
for test2 in antagonists:
if kernel in plot_data[test2]:
del plot_data[test2][kernel]
return plot_data