def testListValue(self):
page0 = self.pages[0]
page1 = self.pages[1]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = list_of_scalar_values.ListOfScalarValues(page0, 'b', 'seconds', [2, 2])
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page1)
v1 = list_of_scalar_values.ListOfScalarValues(page1, 'b', 'seconds', [3, 3])
results.AddValue(v1)
results.DidRunPage(page1)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
b_summary = list_of_scalar_values.ListOfScalarValues(
None, 'b', 'seconds', [2, 2, 3, 3])
self.assertEquals(3, len(values))
self.assertIn(v0, values)
self.assertIn(v1, values)
self.assertIn(b_summary, values)
def testBasicSummary(self):
page0 = self.pages[0]
page1 = self.pages[1]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3)
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page1)
v1 = scalar.ScalarValue(page1, 'a', 'seconds', 7)
results.AddValue(v1)
results.DidRunPage(page1)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
v0_list = list_of_scalar_values.ListOfScalarValues(
page0, 'a', 'seconds', [3])
v1_list = list_of_scalar_values.ListOfScalarValues(
page1, 'a', 'seconds', [7])
merged_value = list_of_scalar_values.ListOfScalarValues(
None, 'a', 'seconds', [3, 7])
self.assertEquals(3, len(values))
self.assertIn(v0_list, values)
self.assertIn(v1_list, values)
self.assertIn(merged_value, values)
def testPageRunsTwice(self):
page0 = self.pages[0]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'b', 'seconds', 2)
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page0)
v1 = scalar.ScalarValue(page0, 'b', 'seconds', 3)
results.AddValue(v1)
results.DidRunPage(page0)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
page0_aggregated = list_of_scalar_values.ListOfScalarValues(
page0, 'b', 'seconds', [2, 3])
b_summary = list_of_scalar_values.ListOfScalarValues(
None, 'b', 'seconds', [2, 3])
self.assertEquals(2, len(values))
self.assertIn(page0_aggregated, values)
self.assertIn(b_summary, values)
def testBasicSummaryPassAndFailPage(self):
"""If a page failed, only print summary for individual pages."""
page0 = self.pages[0]
page1 = self.pages[1]
results = self.getPageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3,
improvement_direction=improvement_direction.UP)
results.AddValue(v0)
results.Fail('message')
results.DidRunPage(page0)
results.WillRunPage(page1)
v2 = scalar.ScalarValue(page1, 'a', 'seconds', 7,
improvement_direction=improvement_direction.UP)
results.AddValue(v2)
results.DidRunPage(page1)
summary = summary_module.Summary(results)
values = summary.interleaved_computed_per_page_values_and_summaries
v0_list = list_of_scalar_values.ListOfScalarValues(
page0, 'a', 'seconds', [3],
improvement_direction=improvement_direction.UP)
v2_list = list_of_scalar_values.ListOfScalarValues(
page1, 'a', 'seconds', [7],
improvement_direction=improvement_direction.UP)
self.assertEquals(2, len(values))
self.assertIn(v0_list, values)
self.assertIn(v2_list, values)
def testBasicSummaryPassAndFailPage(self):
"""If a page failed, only print summary for individual pages."""
page0 = self.pages[0]
page1 = self.pages[1]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3)
results.AddValue(v0)
v1 = failure.FailureValue.FromMessage(page0, 'message')
results.AddValue(v1)
results.DidRunPage(page0)
results.WillRunPage(page1)
v2 = scalar.ScalarValue(page1, 'a', 'seconds', 7)
results.AddValue(v2)
results.DidRunPage(page1)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
v0_list = list_of_scalar_values.ListOfScalarValues(
page0, 'a', 'seconds', [3])
v2_list = list_of_scalar_values.ListOfScalarValues(
page1, 'a', 'seconds', [7])
self.assertEquals(2, len(values))
self.assertIn(v0_list, values)
self.assertIn(v2_list, values)
def testBasicSummaryWithOnlyOnePage(self):
page0 = self.pages[0]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0,
'a',
'seconds',
3,
improvement_direction=improvement_direction.UP)
results.AddValue(v0)
results.DidRunPage(page0)
summary = summary_module.Summary(results)
values = summary.interleaved_computed_per_page_values_and_summaries
v0_list = list_of_scalar_values.ListOfScalarValues(
page0,
'a',
'seconds', [3],
improvement_direction=improvement_direction.UP)
merged_list = list_of_scalar_values.ListOfScalarValues(
None,
'a',
'seconds', [3],
improvement_direction=improvement_direction.UP)
self.assertEquals(2, len(values))
self.assertIn(v0_list, values)
self.assertIn(merged_list, values)
def testRepeatedPagesetOneIterationOnePageFails(self):
"""Page fails on one iteration, no averaged results should print."""
page0 = self.pages[0]
page1 = self.pages[1]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0,
'a',
'seconds',
3,
improvement_direction=improvement_direction.UP)
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page1)
v1 = scalar.ScalarValue(page1,
'a',
'seconds',
7,
improvement_direction=improvement_direction.UP)
results.AddValue(v1)
results.Fail('message')
results.DidRunPage(page1)
results.WillRunPage(page0)
v3 = scalar.ScalarValue(page0,
'a',
'seconds',
4,
improvement_direction=improvement_direction.UP)
results.AddValue(v3)
results.DidRunPage(page0)
results.WillRunPage(page1)
v4 = scalar.ScalarValue(page1,
'a',
'seconds',
8,
improvement_direction=improvement_direction.UP)
results.AddValue(v4)
results.DidRunPage(page1)
summary = summary_module.Summary(results)
values = summary.interleaved_computed_per_page_values_and_summaries
page0_aggregated = list_of_scalar_values.ListOfScalarValues(
page0,
'a',
'seconds', [3, 4],
improvement_direction=improvement_direction.UP)
page1_aggregated = list_of_scalar_values.ListOfScalarValues(
page1,
'a',
'seconds', [7, 8],
improvement_direction=improvement_direction.UP)
self.assertEquals(2, len(values))
self.assertIn(page0_aggregated, values)
self.assertIn(page1_aggregated, values)
def testPageRunsTwice(self):
page0 = self.pages[0]
results = self.getPageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'b', 'seconds', 2,
improvement_direction=improvement_direction.UP)
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page0)
v1 = scalar.ScalarValue(page0, 'b', 'seconds', 3,
improvement_direction=improvement_direction.UP)
results.AddValue(v1)
results.DidRunPage(page0)
summary = summary_module.Summary(results)
values = summary.interleaved_computed_per_page_values_and_summaries
page0_aggregated = list_of_scalar_values.ListOfScalarValues(
page0, 'b', 'seconds', [2, 3],
improvement_direction=improvement_direction.UP)
b_summary = list_of_scalar_values.ListOfScalarValues(
None, 'b', 'seconds', [2, 3],
improvement_direction=improvement_direction.UP)
self.assertEquals(2, len(values))
self.assertIn(page0_aggregated, values)
self.assertIn(b_summary, values)
def testListValue(self):
page0 = self.pages[0]
page1 = self.pages[1]
results = self.getPageTestResults()
results.WillRunPage(page0)
v0 = list_of_scalar_values.ListOfScalarValues(
page0, 'b', 'seconds', [2, 2],
improvement_direction=improvement_direction.UP)
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page1)
v1 = list_of_scalar_values.ListOfScalarValues(
page1, 'b', 'seconds', [3, 3],
improvement_direction=improvement_direction.UP)
results.AddValue(v1)
results.DidRunPage(page1)
summary = summary_module.Summary(results)
values = summary.interleaved_computed_per_page_values_and_summaries
b_summary = list_of_scalar_values.ListOfScalarValues(
None, 'b', 'seconds', [2, 2, 3, 3], std=0.0,
improvement_direction=improvement_direction.UP)
self.assertEquals(3, len(values))
self.assertIn(v0, values)
self.assertIn(v1, values)
self.assertIn(b_summary, values)
def testPagesetRepeat(self):
story_set = story_module.StorySet()
# TODO(eakuefner): Factor this out after flattening page ref in Value
blank_story = DummyLocalStory(TestSharedPageState, name='blank')
green_story = DummyLocalStory(TestSharedPageState, name='green')
story_set.AddStory(blank_story)
story_set.AddStory(green_story)
self.options.pageset_repeat = 2
self.options.output_formats = []
results = results_options.CreateResults(
EmptyMetadataForTest(), self.options)
story_runner.Run(_Measurement(), story_set, self.options, results,
metadata=EmptyMetadataForTest())
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
blank_value = list_of_scalar_values.ListOfScalarValues(
blank_story, 'metric', 'unit', [1, 3],
improvement_direction=improvement_direction.UP)
green_value = list_of_scalar_values.ListOfScalarValues(
green_story, 'metric', 'unit', [2, 4],
improvement_direction=improvement_direction.UP)
merged_value = list_of_scalar_values.ListOfScalarValues(
None, 'metric', 'unit',
[1, 3, 2, 4], std=math.sqrt(2), # Pooled standard deviation.
improvement_direction=improvement_direction.UP)
self.assertEquals(4, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(3, len(values))
self.assertIn(blank_value, values)
self.assertIn(green_value, values)
self.assertIn(merged_value, values)
def testHistogram(self):
page0 = self.pages[0]
page1 = self.pages[1]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = histogram.HistogramValue(
page0, 'a', 'units',
raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}',
important=False)
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page1)
v1 = histogram.HistogramValue(
page1, 'a', 'units',
raw_value_json='{"buckets": [{"low": 2, "high": 3, "count": 1}]}',
important=False)
results.AddValue(v1)
results.DidRunPage(page1)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
self.assertEquals(2, len(values))
self.assertIn(v0, values)
self.assertIn(v1, values)
def ResultsAsChartDict(benchmark_metadata, page_specific_values,
summary_values):
"""Produces a dict for serialization to Chart JSON format from raw values.
Chart JSON is a transformation of the basic Telemetry JSON format that
removes the page map, summarizes the raw values, and organizes the results
by chart and trace name. This function takes the key pieces of data needed to
perform this transformation (namely, lists of values and a benchmark metadata
object) and processes them into a dict which can be serialized using the json
module.
Design doc for schema: http://goo.gl/kOtf1Y
Args:
page_specific_values: list of page-specific values
summary_values: list of summary values
benchmark_metadata: a benchmark.BenchmarkMetadata object
Returns:
A Chart JSON dict corresponding to the given data.
"""
summary = summary_module.Summary(page_specific_values)
values = itertools.chain(
summary.interleaved_computed_per_page_values_and_summaries,
summary_values)
charts = collections.defaultdict(dict)
for value in values:
if value.page:
chart_name, trace_name = (
value.GetChartAndTraceNameForPerPageResult())
else:
chart_name, trace_name = (
value.GetChartAndTraceNameForComputedSummaryResult(None))
if chart_name == trace_name:
trace_name = 'summary'
if value.tir_label:
chart_name = value.tir_label + '@@' + chart_name
# This intentionally overwrites the trace if it already exists because this
# is expected of output from the buildbots currently.
# See: crbug.com/413393
charts[chart_name][trace_name] = value.AsDict()
result_dict = {
'format_version': '0.1',
'next_version': '0.2',
# TODO(sullivan): benchmark_name, benchmark_description, and
# trace_rerun_options should be removed when incrementing format_version
# to 0.1.
'benchmark_name': benchmark_metadata.name,
'benchmark_description': benchmark_metadata.description,
'trace_rerun_options': benchmark_metadata.rerun_options,
'benchmark_metadata': benchmark_metadata.AsDict(),
'charts': charts,
}
return result_dict
def SummarizePageSpecificValues(results):
"""Summarize results appropriately for TBM and legacy benchmarks.
For benchmarks that are timeline-based, we need to summarize not once, but
twice, once by name and grouping_label (default) and again by name only. But
for benchmarks that are not timeline-based, we only summarize once by name.
"""
# Default summary uses merge_values.DefaultKeyFunc to summarize both by name
# and grouping_label.
summary = summary_module.Summary(results)
values = summary.interleaved_computed_per_page_values_and_summaries
if any(v.grouping_label for v in results.IterAllLegacyValues()):
summary_by_name_only = summary_module.Summary(
results, key_func=lambda v: v.name)
values.extend(summary_by_name_only.
interleaved_computed_per_page_values_and_summaries)
return values
def testPagesetRepeat(self):
story_set = story_module.StorySet()
# TODO(eakuefner): Factor this out after flattening page ref in Value
blank_story = DummyLocalStory(TestSharedPageState, name='blank')
green_story = DummyLocalStory(TestSharedPageState, name='green')
story_set.AddStory(blank_story)
story_set.AddStory(green_story)
class Measurement(page_test.PageTest):
i = 0
def RunPage(self, page, _, results):
self.i += 1
results.AddValue(
scalar.ScalarValue(
page,
'metric',
'unit',
self.i,
improvement_direction=improvement_direction.UP))
def ValidateAndMeasurePage(self, page, tab, results):
pass
self.options.page_repeat = 1
self.options.pageset_repeat = 2
self.options.output_formats = []
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
story_runner.Run(Measurement(), story_set, self.options, results)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
blank_value = list_of_scalar_values.ListOfScalarValues(
blank_story,
'metric',
'unit', [1, 3],
improvement_direction=improvement_direction.UP)
green_value = list_of_scalar_values.ListOfScalarValues(
green_story,
'metric',
'unit', [2, 4],
improvement_direction=improvement_direction.UP)
merged_value = list_of_scalar_values.ListOfScalarValues(
None,
'metric',
'unit', [1, 2, 3, 4],
improvement_direction=improvement_direction.UP)
self.assertEquals(4, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(3, len(values))
self.assertIn(blank_value, values)
self.assertIn(green_value, values)
self.assertIn(merged_value, values)
def testBasicSummaryNonuniformResults(self):
page0 = self.pages[0]
page1 = self.pages[1]
page2 = self.pages[2]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3)
results.AddValue(v0)
v1 = scalar.ScalarValue(page0, 'b', 'seconds', 10)
results.AddValue(v1)
results.DidRunPage(page0)
results.WillRunPage(page1)
v2 = scalar.ScalarValue(page1, 'a', 'seconds', 3)
results.AddValue(v2)
v3 = scalar.ScalarValue(page1, 'b', 'seconds', 10)
results.AddValue(v3)
results.DidRunPage(page1)
results.WillRunPage(page2)
v4 = scalar.ScalarValue(page2, 'a', 'seconds', 7)
results.AddValue(v4)
# Note, page[2] does not report a 'b' metric.
results.DidRunPage(page2)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
v0_list = list_of_scalar_values.ListOfScalarValues(
page0, 'a', 'seconds', [3])
v1_list = list_of_scalar_values.ListOfScalarValues(
page0, 'b', 'seconds', [10])
v2_list = list_of_scalar_values.ListOfScalarValues(
page1, 'a', 'seconds', [3])
v3_list = list_of_scalar_values.ListOfScalarValues(
page1, 'b', 'seconds', [10])
v4_list = list_of_scalar_values.ListOfScalarValues(
page2, 'a', 'seconds', [7])
a_summary = list_of_scalar_values.ListOfScalarValues(
None, 'a', 'seconds', [3, 3, 7])
b_summary = list_of_scalar_values.ListOfScalarValues(
None, 'b', 'seconds', [10, 10])
self.assertEquals(7, len(values))
self.assertIn(v0_list, values)
self.assertIn(v1_list, values)
self.assertIn(v2_list, values)
self.assertIn(v3_list, values)
self.assertIn(v4_list, values)
self.assertIn(a_summary, values)
self.assertIn(b_summary, values)
def SummarizePageSpecificValues(page_specific_values):
"""Summarize results appropriately for TBM and legacy benchmarks.
For benchmarks that are timeline-based, we need to summarize not once, but
twice, once by name and tir_label (default) and again by name only. But for
benchmarks that are not timeline-based, since no tir_labels are set, we will
end up duplicating values.
Thus, we only want to summarize once if the benchmark is not timeline-based,
but twice, using the two different key functions, otherwise.
"""
# Default summary uses merge_values.DefaultKeyFunc to summarize both by name
# and tir_label.
summary = summary_module.Summary(page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
if any(v.tir_label for v in page_specific_values):
summary_by_name_only = summary_module.Summary(
page_specific_values, key_func=lambda v: v.name)
values.extend(summary_by_name_only.
interleaved_computed_per_page_values_and_summaries)
return values
def testBasicSummary(self):
page0 = self.pages[0]
page1 = self.pages[1]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0,
'a',
'seconds',
3,
improvement_direction=improvement_direction.UP)
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page1)
v1 = scalar.ScalarValue(page1,
'a',
'seconds',
7,
improvement_direction=improvement_direction.UP)
results.AddValue(v1)
results.DidRunPage(page1)
summary = summary_module.Summary(results)
values = summary.interleaved_computed_per_page_values_and_summaries
v0_list = list_of_scalar_values.ListOfScalarValues(
page0,
'a',
'seconds', [3],
improvement_direction=improvement_direction.UP)
v1_list = list_of_scalar_values.ListOfScalarValues(
page1,
'a',
'seconds', [7],
improvement_direction=improvement_direction.UP)
# Std is 0 because we only have one measurement per page.
merged_value = list_of_scalar_values.ListOfScalarValues(
None,
'a',
'seconds', [3, 7],
std=0.0,
improvement_direction=improvement_direction.UP)
self.assertEquals(3, len(values))
self.assertIn(v0_list, values)
self.assertIn(v1_list, values)
self.assertIn(merged_value, values)
def testRepeatedPages(self):
page0 = self.pages[0]
page1 = self.pages[1]
results = self.getPageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3,
improvement_direction=improvement_direction.UP)
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page0)
v2 = scalar.ScalarValue(page0, 'a', 'seconds', 4,
improvement_direction=improvement_direction.UP)
results.AddValue(v2)
results.DidRunPage(page0)
results.WillRunPage(page1)
v1 = scalar.ScalarValue(page1, 'a', 'seconds', 7,
improvement_direction=improvement_direction.UP)
results.AddValue(v1)
results.DidRunPage(page1)
results.WillRunPage(page1)
v3 = scalar.ScalarValue(page1, 'a', 'seconds', 8,
improvement_direction=improvement_direction.UP)
results.AddValue(v3)
results.DidRunPage(page1)
summary = summary_module.Summary(results)
values = summary.interleaved_computed_per_page_values_and_summaries
page0_aggregated = list_of_scalar_values.ListOfScalarValues(
page0, 'a', 'seconds', [3, 4],
improvement_direction=improvement_direction.UP)
page1_aggregated = list_of_scalar_values.ListOfScalarValues(
page1, 'a', 'seconds', [7, 8],
improvement_direction=improvement_direction.UP)
# Std is computed using pooled standard deviation.
a_summary = list_of_scalar_values.ListOfScalarValues(
None, 'a', 'seconds', [3, 4, 7, 8], std=math.sqrt(0.5),
improvement_direction=improvement_direction.UP)
self.assertEquals(3, len(values))
self.assertIn(page0_aggregated, values)
self.assertIn(page1_aggregated, values)
self.assertIn(a_summary, values)
def testBasicSummaryWithOnlyOnePage(self):
page0 = self.pages[0]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3)
results.AddValue(v0)
results.DidRunPage(page0)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
v0_list = list_of_scalar_values.ListOfScalarValues(
None, 'a', 'seconds', [3])
self.assertEquals([v0_list], values)
def testRunStoryDisabledStory(self):
story_set = story_module.StorySet()
story_one = DummyLocalStory(TestSharedPageState, name='one')
story_set.AddStory(story_one)
results = results_options.CreateResults(
EmptyMetadataForTest(), self.options)
story_runner.Run(_Measurement(), story_set, self.options, results,
expectations=_DisableStoryExpectations(),
metadata=EmptyMetadataForTest())
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
self.assertEquals(1, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(1, GetNumberOfSkippedPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(0, len(values))
def PrintSummary(self):
"""Print summary data in a format expected by buildbot for perf dashboards.
If any failed pages exist, only output individual page results, and do
not output any average data.
"""
# Print out the list of unique pages.
perf_tests_helper.PrintPages(
[page.display_name for page in self.pages_that_succeeded])
summary = summary_module.Summary(self.all_page_specific_values,
self.had_errors_or_failures)
for value in summary.interleaved_computed_per_page_values_and_summaries:
if value.page:
self._PrintComputedPerPageValue(value)
else:
self._PrintComputedSummaryValue(value)
self._PrintOverallResults()
def testSummaryUsesKeyFunc(self):
page0 = self.pages[0]
page1 = self.pages[1]
results = self.getPageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'a', 'seconds', 20,
improvement_direction=improvement_direction.UP)
results.AddValue(v0)
v1 = scalar.ScalarValue(page0, 'b', 'seconds', 42,
improvement_direction=improvement_direction.UP)
results.AddValue(v1)
results.DidRunPage(page0)
results.WillRunPage(page1)
v2 = scalar.ScalarValue(page1, 'a', 'seconds', 20,
improvement_direction=improvement_direction.UP)
results.AddValue(v2)
v3 = scalar.ScalarValue(page1, 'b', 'seconds', 42,
improvement_direction=improvement_direction.UP)
results.AddValue(v3)
results.DidRunPage(page1)
summary = summary_module.Summary(
results, key_func=lambda v: True)
values = summary.interleaved_computed_per_page_values_and_summaries
v0_list = list_of_scalar_values.ListOfScalarValues(
page0, 'a', 'seconds', [20, 42],
improvement_direction=improvement_direction.UP)
v2_list = list_of_scalar_values.ListOfScalarValues(
page1, 'a', 'seconds', [20, 42],
improvement_direction=improvement_direction.UP)
# Std is computed using pooled standard deviation.
merged_value = list_of_scalar_values.ListOfScalarValues(
None, 'a', 'seconds', [20, 42, 20, 42], std=math.sqrt(242.0),
improvement_direction=improvement_direction.UP)
self.assertEquals(3, len(values))
self.assertIn(v0_list, values)
self.assertIn(v2_list, values)
self.assertIn(merged_value, values)
def testRepeatedPages(self):
page0 = self.pages[0]
page1 = self.pages[1]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3)
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page0)
v2 = scalar.ScalarValue(page0, 'a', 'seconds', 4)
results.AddValue(v2)
results.DidRunPage(page0)
results.WillRunPage(page1)
v1 = scalar.ScalarValue(page1, 'a', 'seconds', 7)
results.AddValue(v1)
results.DidRunPage(page1)
results.WillRunPage(page1)
v3 = scalar.ScalarValue(page1, 'a', 'seconds', 8)
results.AddValue(v3)
results.DidRunPage(page1)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
page0_aggregated = list_of_scalar_values.ListOfScalarValues(
page0, 'a', 'seconds', [3, 4])
page1_aggregated = list_of_scalar_values.ListOfScalarValues(
page1, 'a', 'seconds', [7, 8])
a_summary = list_of_scalar_values.ListOfScalarValues(
None, 'a', 'seconds', [3, 4, 7, 8])
self.assertEquals(3, len(values))
self.assertIn(page0_aggregated, values)
self.assertIn(page1_aggregated, values)
self.assertIn(a_summary, values)
def testBasicSummaryNonuniformResults(self):
page0 = self.pages[0]
page1 = self.pages[1]
page2 = self.pages[2]
results = self.getPageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3,
improvement_direction=improvement_direction.UP)
results.AddValue(v0)
v1 = scalar.ScalarValue(page0, 'b', 'seconds', 10,
improvement_direction=improvement_direction.UP)
results.AddValue(v1)
results.DidRunPage(page0)
results.WillRunPage(page1)
v2 = scalar.ScalarValue(page1, 'a', 'seconds', 3,
improvement_direction=improvement_direction.UP)
results.AddValue(v2)
v3 = scalar.ScalarValue(page1, 'b', 'seconds', 10,
improvement_direction=improvement_direction.UP)
results.AddValue(v3)
results.DidRunPage(page1)
results.WillRunPage(page2)
v4 = scalar.ScalarValue(page2, 'a', 'seconds', 7,
improvement_direction=improvement_direction.UP)
results.AddValue(v4)
# Note, page[2] does not report a 'b' metric.
results.DidRunPage(page2)
summary = summary_module.Summary(results)
values = summary.interleaved_computed_per_page_values_and_summaries
v0_list = list_of_scalar_values.ListOfScalarValues(
page0, 'a', 'seconds', [3],
improvement_direction=improvement_direction.UP)
v1_list = list_of_scalar_values.ListOfScalarValues(
page0, 'b', 'seconds', [10],
improvement_direction=improvement_direction.UP)
v2_list = list_of_scalar_values.ListOfScalarValues(
page1, 'a', 'seconds', [3],
improvement_direction=improvement_direction.UP)
v3_list = list_of_scalar_values.ListOfScalarValues(
page1, 'b', 'seconds', [10],
improvement_direction=improvement_direction.UP)
v4_list = list_of_scalar_values.ListOfScalarValues(
page2, 'a', 'seconds', [7],
improvement_direction=improvement_direction.UP)
# Std is 0 because we only have one measurement per page.
a_summary = list_of_scalar_values.ListOfScalarValues(
None, 'a', 'seconds', [3, 3, 7], std=0.0,
improvement_direction=improvement_direction.UP)
b_summary = list_of_scalar_values.ListOfScalarValues(
None, 'b', 'seconds', [10, 10], std=0.0,
improvement_direction=improvement_direction.UP)
self.assertEquals(7, len(values))
self.assertIn(v0_list, values)
self.assertIn(v1_list, values)
self.assertIn(v2_list, values)
self.assertIn(v3_list, values)
self.assertIn(v4_list, values)
self.assertIn(a_summary, values)
self.assertIn(b_summary, values)
