Exemplo n.º 1
0
    def analyze_variants_all_tests_1_platform(self, platform, vary):
        # generate one graph image for results of all benchmarks
        # on one platform and one kernel, comparing effects of
        # two or more combos of kernel options (test run attributes)
        #   (numa_fake,stale_page,kswapd_merge,sched_idle, etc)
        kernel = self.cgiform.getvalue('kernel', 'some_kernel')
        self.passthru.append('kernel=%s' % kernel)

        # two or more vary_groups, one for each plotted line,
        # each group begins with vary= and ends with next  &
        # each group has comma-separated list of test attribute key=val pairs
        #    eg   vary=keyval1,keyval2&vary=keyval3,keyval4
        vary_groups = [
            dict(pair.split('=', 1) for pair in vary_group.split(','))
            for vary_group in vary
        ]

        test = self.benchmarks[0]  # pick any test in all jobs
        kernels, test_tag = self.jobs_selector(test, self.job_table,
                                               self.kernel_dates)

        linekeys = {}
        plot_data = {}
        baselines = {}
        for i, vary_group in enumerate(vary_groups):
            group_attributes = self.test_attributes.copy()
            group_attributes.update(vary_group)
            linekey = ','.join('%s=%s' % (attr, vary_group[attr])
                               for attr in vary_group)
            linekeys[i] = linekey
            data = {}
            for benchmark in self.benchmarks:
                metric = perf.benchmark_main_metric(benchmark)
                runs = collect_testruns(self.job_table[kernel],
                                        benchmark + test_tag, group_attributes,
                                        self.platforms_filter, 'by_hosts'
                                        in self.toggles, self.no_antag)
                vals = []
                for testrunx in runs[platform]:
                    vals += perf.get_metric_at_point([testrunx], metric)
                if vals:
                    if benchmark not in baselines:
                        baselines[benchmark], stddev = plotgraph.avg_dev(vals)
                    vals = [val / baselines[benchmark] for val in vals]
                    data[benchmark] = vals
            plot_data[i] = data

        title = "%s on %s" % (kernel, platform)
        for attr in self.test_attributes:
            title += ', %s=%s' % (attr, self.test_attributes[attr])
        if 'table' in self.cgiform:
            self.table_for_variants_all_tests(title,
                                              plot_data,
                                              linekeys,
                                              range(len(linekeys)),
                                              filtered_passthru=self.passthru,
                                              test_tag=test_tag)
        else:
            graph_variants_all_tests(title, plot_data, linekeys, self.size,
                                     'dark' in self.toggles)
Exemplo n.º 2
0
def table_testrun_details(runs, metric, tko_server, show_attrs):
    print "<table border=1 cellpadding=4 cellspacing=0>"
    print "<tr><td> %s metric </td>" % metric
    print "<td> Job label </td> <td> Job tag </td> <td> Run results </td>"
    print "<td> Started_time </td>"
    if show_attrs:
        print "<td> Test attributes </td>"
    print "</tr>\n"

    for testrunx in runs:
        print "<tr> <td>",
        vals = perf.get_metric_at_point([testrunx], metric)
        for v in vals:
            print "%.4g&nbsp;" % v,
        print "</td>"
        row = get_testrun_context(testrunx)
        row[2] = ( "<a href='//%s/results/%s/%s/results/keyval'> %s </a>"
                   % (tko_server, row[1], row[2], row[2]) )
        for v in row:
            print "<td> %s </td>" % v
        if show_attrs:
            attrs = get_test_attributes(testrunx)
            print "<td>",
            for attr in sorted(attrs.keys()):
                if attr == "sysinfo-cmdline": continue
                if attr[:4] == "svs-": continue
                val = attrs[attr]
                if len(val) > 40:
                    val = val[:40-3] + "..."
                print "%s=%s &nbsp; &nbsp; " % (attr, val)
            print "</td>"
        print "</tr>\n"
    print "</table>"
Exemplo n.º 3
0
def table_testrun_details(runs, metric, tko_server, show_attrs):
    print "<table border=1 cellpadding=4 cellspacing=0>"
    print "<tr><td> %s metric </td>" % metric
    print "<td> Job label </td> <td> Job tag </td> <td> Run results </td>"
    print "<td> Started_time </td>"
    if show_attrs:
        print "<td> Test attributes </td>"
    print "</tr>\n"

    for testrunx in runs:
        print "<tr> <td>",
        vals = perf.get_metric_at_point([testrunx], metric)
        for v in vals:
            print "%.4g&nbsp;" % v,
        print "</td>"
        row = get_testrun_context(testrunx)
        row[2] = ("<a href='//%s/results/%s/%s/results/keyval'> %s </a>" %
                  (tko_server, row[1], row[2], row[2]))
        for v in row:
            print "<td> %s </td>" % v
        if show_attrs:
            attrs = get_test_attributes(testrunx)
            print "<td>",
            for attr in sorted(attrs.keys()):
                if attr == "sysinfo-cmdline": continue
                if attr[:4] == "svs-": continue
                val = attrs[attr]
                if len(val) > 40:
                    val = val[:40 - 3] + "..."
                print "%s=%s &nbsp; &nbsp; " % (attr, val)
            print "</td>"
        print "</tr>\n"
    print "</table>"
Exemplo n.º 4
0
    def analyze_variants_all_tests_1_platform(self, platform, vary):
        # generate one graph image for results of all benchmarks
        # on one platform and one kernel, comparing effects of
        # two or more combos of kernel options (test run attributes)
        #   (numa_fake,stale_page,kswapd_merge,sched_idle, etc)
        kernel = self.cgiform.getvalue('kernel', 'some_kernel')
        self.passthru.append('kernel=%s' % kernel)

        # two or more vary_groups, one for each plotted line,
        # each group begins with vary= and ends with next  &
        # each group has comma-separated list of test attribute key=val pairs
        #    eg   vary=keyval1,keyval2&vary=keyval3,keyval4
        vary_groups = [dict(pair.split('=',1) for pair
                            in vary_group.split(','))
                       for vary_group in vary]

        test = self.benchmarks[0]  # pick any test in all jobs
        kernels, test_tag = self.jobs_selector(test, self.job_table,
                                               self.kernel_dates)

        linekeys = {}
        plot_data = {}
        baselines = {}
        for i, vary_group in enumerate(vary_groups):
            group_attributes = self.test_attributes.copy()
            group_attributes.update(vary_group)
            linekey = ','.join('%s=%s' % (attr, vary_group[attr])
                               for attr in vary_group)
            linekeys[i] = linekey
            data = {}
            for benchmark in self.benchmarks:
                metric = perf.benchmark_main_metric(benchmark)
                runs = collect_testruns(self.job_table[kernel],
                                        benchmark+test_tag,
                                        group_attributes,
                                        self.platforms_filter,
                                        'by_hosts' in self.toggles,
                                        self.no_antag)
                vals = []
                for testrunx in runs[platform]:
                    vals += perf.get_metric_at_point([testrunx], metric)
                if vals:
                    if benchmark not in baselines:
                        baselines[benchmark], stddev = plotgraph.avg_dev(vals)
                    vals = [val/baselines[benchmark] for val in vals]
                    data[benchmark] = vals
            plot_data[i] = data

        title  = "%s on %s" % (kernel, platform)
        for attr in self.test_attributes:
            title += ', %s=%s' % (attr, self.test_attributes[attr])
        if 'table' in self.cgiform:
            self.table_for_variants_all_tests(title, plot_data, linekeys,
                               range(len(linekeys)),
                               filtered_passthru=self.passthru,
                               test_tag=test_tag)
        else:
            graph_variants_all_tests(title, plot_data, linekeys,
                                     self.size, 'dark' in self.toggles)
Exemplo n.º 5
0
def collect_raw_scores(runs, metric):
    # get unscaled scores of test runs for 1 test on certain jobs
    #   arrange them by platform type
    platform_scores = {}  # platform --> list of perf scores
    for platform in runs:
        vals = perf.get_metric_at_point(runs[platform], metric)
        if vals:
            platform_scores[platform] = vals
    return platform_scores
Exemplo n.º 6
0
def collect_raw_scores(runs, metric):
    # get unscaled scores of test runs for 1 test on certain jobs
    #   arrange them by platform type
    platform_scores = {}  # platform --> list of perf scores
    for platform in runs:
        vals = perf.get_metric_at_point(runs[platform], metric)
        if vals:
            platform_scores[platform] = vals
    return platform_scores
Exemplo n.º 7
0
 def collect_scaled_scores(metric):
     # get scores of test runs for 1 test on some kernels and platforms
     # optionally make relative to first kernel on that platform 
     # arrange by plotline (ie platform) for gnuplot
     plot_data = {}  # platform --> (kernel --> list of perf scores)
     baseline = {}
     for kernel in sorted(test_runs.keys()):
         for platform in test_runs[kernel]:
             vals = perf.get_metric_at_point(test_runs[kernel][platform], metric)
             if vals:
                 if relative:
                     if platform not in baseline:
                         baseline[platform], std = plotgraph.avg_dev(vals)
                     vals = [v/baseline[platform] for v in vals]
                 pdp = plot_data.setdefault(platform, {})
                 pdp.setdefault(kernel, []).extend(vals)
     return plot_data
Exemplo n.º 8
0
def collect_scaled_scores(metric, test_runs, regressed_platforms, relative):
    # get scores of test runs for 1 test on some kernels and platforms
    # optionally make relative to oldest (?) kernel on that platform
    # arrange by plotline (ie platform) for gnuplot
    plot_data = {}  # platform --> (kernel --> list of perf scores)
    baseline = {}
    for kernel in sorted(test_runs.keys()):
        for platform in test_runs[kernel]:
            if not (regressed_platforms is None
                    or platform in regressed_platforms):
                continue  # delete results for uninteresting platforms
            vals = perf.get_metric_at_point(test_runs[kernel][platform],
                                            metric)
            if vals:
                if relative:
                    if platform not in baseline:
                        baseline[platform], std = plotgraph.avg_dev(vals)
                    vals = [v / baseline[platform] for v in vals]
                pdp = plot_data.setdefault(platform, {})
                pdp.setdefault(kernel, []).extend(vals)
    return plot_data
Exemplo n.º 9
0
    def testrun_details_for_1_test_kernel_platform(test, metric, platform):
        kernel   = form.getvalue('kernel')
        show_attrs  = 'attrs' in form
        print "Content-Type: text/html\n\n<html><body>"
        print "<h3> %s %s:&nbsp; %s%s, Kernel %s on %s </h3>" % (
            test_group, kernel_legend, test.capitalize(), suite_notes, kernel, platform)
        print "<table border=1 cellpadding=4>"
        print "<tr><td> Test_idx </td> <td> %s metric </td>" % metric
        print "<td> Job tag </td> <td> Subdir </td> <td> Job label </td>"
        print "<td> Started_time </td>"
        if show_attrs:
            print "<td> Test attributes </td>"
        print "</tr>\n"

        for testrunx in test_runs[kernel][platform]:
            print "<tr> <td>", testrunx, "</td>"
            print "<td>",
            vals = perf.get_metric_at_point([testrunx], metric)
            for v in vals:
                print "%.4g" % v,
            print "</td>"
            row = get_testrun_context(testrunx)
            print ( "<td> <a href='//%s/results/%s/%s/results'> %s </a></td>" 
                    % (results_server, row[0], row[1], row[0]) )
            for v in row[1:]:
                 print "<td> %s </td>" % v
            if show_attrs:
                attrs = perf.get_test_attributes(testrunx)
                print "<td>",
                for attr in attrs:
                     # if attr == "sysinfo-cmdline": continue
                     # if attr[:4] == "svs-": continue
                     val = attrs[attr]
                     if len(val) > 40:
                         val = val[:40-3] + "..."
                     print "%s=%s" % (attr, val)
                print "</td>"
            print "</tr>\n"
        print "</table>"
        print "</body></html>"
Exemplo n.º 10
0
def collect_twoway_scores(metric, antagonists, twoway_runs, relative):
    alone = ''
    plot_data = {}
    for kernel in twoway_runs:
        for test2 in antagonists:
            runs = twoway_runs[kernel].get(test2, [])
            vals = perf.get_metric_at_point(runs, metric)
            plot_data.setdefault(test2, {})
            if vals:
                plot_data[test2][kernel] = vals
        if relative:
            vals = plot_data[alone].get(kernel, [])
            if vals:
                baseline = perf.average(vals)
                for test2 in antagonists:
                    vals = plot_data[test2].get(kernel, [])
                    vals = [val/baseline for val in vals]
                    if vals:
                        plot_data[test2][kernel] = vals
            else:
                for test2 in antagonists:
                    if kernel in plot_data[test2]:
                        del plot_data[test2][kernel]
    return plot_data
Exemplo n.º 11
0
def collect_twoway_scores(metric, antagonists, twoway_runs, relative):
    alone = ''
    plot_data = {}
    for kernel in twoway_runs:
        for test2 in antagonists:
            runs = twoway_runs[kernel].get(test2, [])
            vals = perf.get_metric_at_point(runs, metric)
            plot_data.setdefault(test2, {})
            if vals:
                plot_data[test2][kernel] = vals
        if relative:
            vals = plot_data[alone].get(kernel, [])
            if vals:
                baseline = perf.average(vals)
                for test2 in antagonists:
                    vals = plot_data[test2].get(kernel, [])
                    vals = [val / baseline for val in vals]
                    if vals:
                        plot_data[test2][kernel] = vals
            else:
                for test2 in antagonists:
                    if kernel in plot_data[test2]:
                        del plot_data[test2][kernel]
    return plot_data