def table_testrun_details(runs, metric, tko_server, show_attrs): print "<table border=1 cellpadding=4 cellspacing=0>" print "<tr><td> %s metric </td>" % metric print "<td> Job label </td> <td> Job tag </td> <td> Run results </td>" print "<td> Started_time </td>" if show_attrs: print "<td> Test attributes </td>" print "</tr>\n" for testrunx in runs: print "<tr> <td>", vals = perf.get_metric_at_point([testrunx], metric) for v in vals: print "%.4g " % v, print "</td>" row = get_testrun_context(testrunx) row[2] = ("<a href='//%s/results/%s/%s/results/keyval'> %s </a>" % (tko_server, row[1], row[2], row[2])) for v in row: print "<td> %s </td>" % v if show_attrs: attrs = get_test_attributes(testrunx) print "<td>", for attr in sorted(attrs.keys()): if attr == "sysinfo-cmdline": continue if attr[:4] == "svs-": continue val = attrs[attr] if len(val) > 40: val = val[:40 - 3] + "..." print "%s=%s " % (attr, val) print "</td>" print "</tr>\n" print "</table>"
def analyze_variants_all_tests_1_platform(self, platform, vary): # generate one graph image for results of all benchmarks # on one platform and one kernel, comparing effects of # two or more combos of kernel options (test run attributes) # (numa_fake,stale_page,kswapd_merge,sched_idle, etc) kernel = self.cgiform.getvalue('kernel', 'some_kernel') self.passthru.append('kernel=%s' % kernel) # two or more vary_groups, one for each plotted line, # each group begins with vary= and ends with next & # each group has comma-separated list of test attribute key=val pairs # eg vary=keyval1,keyval2&vary=keyval3,keyval4 vary_groups = [ dict(pair.split('=', 1) for pair in vary_group.split(',')) for vary_group in vary ] test = self.benchmarks[0] # pick any test in all jobs kernels, test_tag = self.jobs_selector(test, self.job_table, self.kernel_dates) linekeys = {} plot_data = {} baselines = {} for i, vary_group in enumerate(vary_groups): group_attributes = self.test_attributes.copy() group_attributes.update(vary_group) linekey = ','.join('%s=%s' % (attr, vary_group[attr]) for attr in vary_group) linekeys[i] = linekey data = {} for benchmark in self.benchmarks: metric = perf.benchmark_main_metric(benchmark) runs = collect_testruns(self.job_table[kernel], benchmark + test_tag, group_attributes, self.platforms_filter, 'by_hosts' in self.toggles, self.no_antag) vals = [] for testrunx in runs[platform]: vals += perf.get_metric_at_point([testrunx], metric) if vals: if benchmark not in baselines: baselines[benchmark], stddev = plotgraph.avg_dev(vals) vals = [val / baselines[benchmark] for val in vals] data[benchmark] = vals plot_data[i] = data title = "%s on %s" % (kernel, platform) for attr in self.test_attributes: title += ', %s=%s' % (attr, self.test_attributes[attr]) if 'table' in self.cgiform: self.table_for_variants_all_tests(title, plot_data, linekeys, range(len(linekeys)), filtered_passthru=self.passthru, test_tag=test_tag) else: graph_variants_all_tests(title, plot_data, linekeys, self.size, 'dark' in self.toggles)
def table_testrun_details(runs, metric, tko_server, show_attrs): print "<table border=1 cellpadding=4 cellspacing=0>" print "<tr><td> %s metric </td>" % metric print "<td> Job label </td> <td> Job tag </td> <td> Run results </td>" print "<td> Started_time </td>" if show_attrs: print "<td> Test attributes </td>" print "</tr>\n" for testrunx in runs: print "<tr> <td>", vals = perf.get_metric_at_point([testrunx], metric) for v in vals: print "%.4g " % v, print "</td>" row = get_testrun_context(testrunx) row[2] = "<a href='//%s/results/%s/%s/results/keyval'> %s </a>" % (tko_server, row[1], row[2], row[2]) for v in row: print "<td> %s </td>" % v if show_attrs: attrs = get_test_attributes(testrunx) print "<td>", for attr in sorted(attrs.keys()): if attr == "sysinfo-cmdline": continue if attr[:4] == "svs-": continue val = attrs[attr] if len(val) > 40: val = val[: 40 - 3] + "..." print "%s=%s " % (attr, val) print "</td>" print "</tr>\n" print "</table>"
def collect_raw_scores(runs, metric): # get unscaled scores of test runs for 1 test on certain jobs # arrange them by platform type platform_scores = {} # platform --> list of perf scores for platform in runs: vals = perf.get_metric_at_point(runs[platform], metric) if vals: platform_scores[platform] = vals return platform_scores
def analyze_variants_all_tests_1_platform(self, platform, vary): # generate one graph image for results of all benchmarks # on one platform and one kernel, comparing effects of # two or more combos of kernel options (test run attributes) # (numa_fake,stale_page,kswapd_merge,sched_idle, etc) kernel = self.cgiform.getvalue("kernel", "some_kernel") self.passthru.append("kernel=%s" % kernel) # two or more vary_groups, one for each plotted line, # each group begins with vary= and ends with next & # each group has comma-separated list of test attribute key=val pairs # eg vary=keyval1,keyval2&vary=keyval3,keyval4 vary_groups = [dict(pair.split("=", 1) for pair in vary_group.split(",")) for vary_group in vary] test = self.benchmarks[0] # pick any test in all jobs kernels, test_tag = self.jobs_selector(test, self.job_table, self.kernel_dates) linekeys = {} plot_data = {} baselines = {} for i, vary_group in enumerate(vary_groups): group_attributes = self.test_attributes.copy() group_attributes.update(vary_group) linekey = ",".join("%s=%s" % (attr, vary_group[attr]) for attr in vary_group) linekeys[i] = linekey data = {} for benchmark in self.benchmarks: metric = perf.benchmark_main_metric(benchmark) runs = collect_testruns( self.job_table[kernel], benchmark + test_tag, group_attributes, self.platforms_filter, "by_hosts" in self.toggles, self.no_antag, ) vals = [] for testrunx in runs[platform]: vals += perf.get_metric_at_point([testrunx], metric) if vals: if benchmark not in baselines: baselines[benchmark], stddev = plotgraph.avg_dev(vals) vals = [val / baselines[benchmark] for val in vals] data[benchmark] = vals plot_data[i] = data title = "%s on %s" % (kernel, platform) for attr in self.test_attributes: title += ", %s=%s" % (attr, self.test_attributes[attr]) if "table" in self.cgiform: self.table_for_variants_all_tests( title, plot_data, linekeys, range(len(linekeys)), filtered_passthru=self.passthru, test_tag=test_tag ) else: graph_variants_all_tests(title, plot_data, linekeys, self.size, "dark" in self.toggles)
def collect_scaled_scores(metric, test_runs, regressed_platforms, relative): # get scores of test runs for 1 test on some kernels and platforms # optionally make relative to oldest (?) kernel on that platform # arrange by plotline (ie platform) for gnuplot plot_data = {} # platform --> (kernel --> list of perf scores) baseline = {} for kernel in sorted(test_runs.keys()): for platform in test_runs[kernel]: if not (regressed_platforms is None or platform in regressed_platforms): continue # delete results for uninteresting platforms vals = perf.get_metric_at_point(test_runs[kernel][platform], metric) if vals: if relative: if platform not in baseline: baseline[platform], std = plotgraph.avg_dev(vals) vals = [v / baseline[platform] for v in vals] pdp = plot_data.setdefault(platform, {}) pdp.setdefault(kernel, []).extend(vals) return plot_data
def collect_twoway_scores(metric, antagonists, twoway_runs, relative): alone = '' plot_data = {} for kernel in twoway_runs: for test2 in antagonists: runs = twoway_runs[kernel].get(test2, []) vals = perf.get_metric_at_point(runs, metric) plot_data.setdefault(test2, {}) if vals: plot_data[test2][kernel] = vals if relative: vals = plot_data[alone].get(kernel, []) if vals: baseline = perf.average(vals) for test2 in antagonists: vals = plot_data[test2].get(kernel, []) vals = [val / baseline for val in vals] if vals: plot_data[test2][kernel] = vals else: for test2 in antagonists: if kernel in plot_data[test2]: del plot_data[test2][kernel] return plot_data
def collect_twoway_scores(metric, antagonists, twoway_runs, relative): alone = "" plot_data = {} for kernel in twoway_runs: for test2 in antagonists: runs = twoway_runs[kernel].get(test2, []) vals = perf.get_metric_at_point(runs, metric) plot_data.setdefault(test2, {}) if vals: plot_data[test2][kernel] = vals if relative: vals = plot_data[alone].get(kernel, []) if vals: baseline = perf.average(vals) for test2 in antagonists: vals = plot_data[test2].get(kernel, []) vals = [val / baseline for val in vals] if vals: plot_data[test2][kernel] = vals else: for test2 in antagonists: if kernel in plot_data[test2]: del plot_data[test2][kernel] return plot_data