def analyze_1_test(self, test, antagonists):
self.passthru.append('test=%s' % test)
metric = self.cgiform.getvalue('metric', '')
if metric:
self.passthru.append('metric=%s' % metric)
else:
metric = perf.benchmark_main_metric(test)
assert metric, "no default metric for test %s" % test
self.kernels, self.test_tag = self.jobs_selector(
test, self.job_table, self.kernel_dates)
self.collect_all_testruns(self.kernels, test + self.test_tag)
if not self.platforms_filter and (metric == '*'
or antagonists is not None):
# choose default platform
self.platforms_filter = all_tested_platforms(self.test_runs)[0:1]
self.passthru.append('platforms=%s' %
','.join(self.platforms_filter))
if antagonists is not None:
antagonists = antagonists.split(',')
if len(antagonists) == 1 and antagonists != ['*']:
self.relative = False
self.analyze_twoway_antagonists_1_test_1_platform(
test, metric, self.platforms_filter[0], antagonists)
elif metric == '*':
platform = self.platforms_filter[0]
self.analyze_all_metrics_1_platform(test, platform)
else:
self.analyze_1_metric_all_platforms(test, metric)
def analyze_variants_all_tests_1_platform(self, platform, vary):
# generate one graph image for results of all benchmarks
# on one platform and one kernel, comparing effects of
# two or more combos of kernel options (test run attributes)
# (numa_fake,stale_page,kswapd_merge,sched_idle, etc)
kernel = self.cgiform.getvalue('kernel', 'some_kernel')
self.passthru.append('kernel=%s' % kernel)
# two or more vary_groups, one for each plotted line,
# each group begins with vary= and ends with next &
# each group has comma-separated list of test attribute key=val pairs
# eg vary=keyval1,keyval2&vary=keyval3,keyval4
vary_groups = [
dict(pair.split('=', 1) for pair in vary_group.split(','))
for vary_group in vary
]
test = self.benchmarks[0] # pick any test in all jobs
kernels, test_tag = self.jobs_selector(test, self.job_table,
self.kernel_dates)
linekeys = {}
plot_data = {}
baselines = {}
for i, vary_group in enumerate(vary_groups):
group_attributes = self.test_attributes.copy()
group_attributes.update(vary_group)
linekey = ','.join('%s=%s' % (attr, vary_group[attr])
for attr in vary_group)
linekeys[i] = linekey
data = {}
for benchmark in self.benchmarks:
metric = perf.benchmark_main_metric(benchmark)
runs = collect_testruns(self.job_table[kernel],
benchmark + test_tag, group_attributes,
self.platforms_filter, 'by_hosts'
in self.toggles, self.no_antag)
vals = []
for testrunx in runs[platform]:
vals += perf.get_metric_at_point([testrunx], metric)
if vals:
if benchmark not in baselines:
baselines[benchmark], stddev = plotgraph.avg_dev(vals)
vals = [val / baselines[benchmark] for val in vals]
data[benchmark] = vals
plot_data[i] = data
title = "%s on %s" % (kernel, platform)
for attr in self.test_attributes:
title += ', %s=%s' % (attr, self.test_attributes[attr])
if 'table' in self.cgiform:
self.table_for_variants_all_tests(title,
plot_data,
linekeys,
range(len(linekeys)),
filtered_passthru=self.passthru,
test_tag=test_tag)
else:
graph_variants_all_tests(title, plot_data, linekeys, self.size,
'dark' in self.toggles)
def analyze_variants_all_tests_1_platform(self, platform, vary):
# generate one graph image for results of all benchmarks
# on one platform and one kernel, comparing effects of
# two or more combos of kernel options (test run attributes)
# (numa_fake,stale_page,kswapd_merge,sched_idle, etc)
kernel = self.cgiform.getvalue("kernel", "some_kernel")
self.passthru.append("kernel=%s" % kernel)
# two or more vary_groups, one for each plotted line,
# each group begins with vary= and ends with next &
# each group has comma-separated list of test attribute key=val pairs
# eg vary=keyval1,keyval2&vary=keyval3,keyval4
vary_groups = [dict(pair.split("=", 1) for pair in vary_group.split(",")) for vary_group in vary]
test = self.benchmarks[0] # pick any test in all jobs
kernels, test_tag = self.jobs_selector(test, self.job_table, self.kernel_dates)
linekeys = {}
plot_data = {}
baselines = {}
for i, vary_group in enumerate(vary_groups):
group_attributes = self.test_attributes.copy()
group_attributes.update(vary_group)
linekey = ",".join("%s=%s" % (attr, vary_group[attr]) for attr in vary_group)
linekeys[i] = linekey
data = {}
for benchmark in self.benchmarks:
metric = perf.benchmark_main_metric(benchmark)
runs = collect_testruns(
self.job_table[kernel],
benchmark + test_tag,
group_attributes,
self.platforms_filter,
"by_hosts" in self.toggles,
self.no_antag,
)
vals = []
for testrunx in runs[platform]:
vals += perf.get_metric_at_point([testrunx], metric)
if vals:
if benchmark not in baselines:
baselines[benchmark], stddev = plotgraph.avg_dev(vals)
vals = [val / baselines[benchmark] for val in vals]
data[benchmark] = vals
plot_data[i] = data
title = "%s on %s" % (kernel, platform)
for attr in self.test_attributes:
title += ", %s=%s" % (attr, self.test_attributes[attr])
if "table" in self.cgiform:
self.table_for_variants_all_tests(
title, plot_data, linekeys, range(len(linekeys)), filtered_passthru=self.passthru, test_tag=test_tag
)
else:
graph_variants_all_tests(title, plot_data, linekeys, self.size, "dark" in self.toggles)
def analyze_1_test(self, test, antagonists):
self.passthru.append("test=%s" % test)
metric = self.cgiform.getvalue("metric", "")
if metric:
self.passthru.append("metric=%s" % metric)
else:
metric = perf.benchmark_main_metric(test)
assert metric, "no default metric for test %s" % test
self.kernels, self.test_tag = self.jobs_selector(test, self.job_table, self.kernel_dates)
self.collect_all_testruns(self.kernels, test + self.test_tag)
if not self.platforms_filter and (metric == "*" or antagonists is not None):
# choose default platform
self.platforms_filter = all_tested_platforms(self.test_runs)[0:1]
self.passthru.append("platforms=%s" % ",".join(self.platforms_filter))
if antagonists is not None:
antagonists = antagonists.split(",")
if len(antagonists) == 1 and antagonists != ["*"]:
self.relative = False
self.analyze_twoway_antagonists_1_test_1_platform(test, metric, self.platforms_filter[0], antagonists)
elif metric == "*":
platform = self.platforms_filter[0]
self.analyze_all_metrics_1_platform(test, platform)
else:
self.analyze_1_metric_all_platforms(test, metric)
