def Run(self, args):
"""Run this test with the given options."""
self.CustomizeBrowserOptions(args.browser_options)
test = self.PageTestClass()()
test.__name__ = self.__class__.__name__
if hasattr(self, '_disabled_strings'):
test._disabled_strings = self._disabled_strings
if hasattr(self, '_enabled_strings'):
test._enabled_strings = self._enabled_strings
ps = self.CreatePageSet(args)
expectations = self.CreateExpectations(ps)
self._DownloadGeneratedProfileArchive(args)
results = page_test_results.PageTestResults()
try:
results = page_runner.Run(test, ps, expectations, args)
except page_test.TestNotSupportedOnPlatformFailure as failure:
logging.warning(str(failure))
results.PrintSummary()
return len(results.failures) + len(results.errors)
def test_add_summary_value_with_page_specified(self):
results = page_test_results.PageTestResults()
results.WillRunPage(self.pages[0])
self.assertRaises(
AssertionError,
lambda: results.AddSummaryValue(scalar.ScalarValue(self.pages[0],
'a', 'units', 3)))
def testFilterIsFirstResult(self):
def AcceptSecondValues(_, is_first_result):
return not is_first_result
results = page_test_results.PageTestResults(
value_can_be_added_predicate=AcceptSecondValues)
# First results (filtered out)
results.WillRunPage(self.pages[0])
results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 7))
results.AddValue(scalar.ScalarValue(self.pages[0], 'b', 'seconds', 8))
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.AddValue(scalar.ScalarValue(self.pages[1], 'a', 'seconds', 5))
results.AddValue(scalar.ScalarValue(self.pages[1], 'd', 'seconds', 6))
results.DidRunPage(self.pages[1])
# Second results
results.WillRunPage(self.pages[0])
results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3))
results.AddValue(scalar.ScalarValue(self.pages[0], 'b', 'seconds', 4))
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
results.AddValue(scalar.ScalarValue(self.pages[1], 'a', 'seconds', 1))
results.AddValue(scalar.ScalarValue(self.pages[1], 'd', 'seconds', 2))
results.DidRunPage(self.pages[1])
results.PrintSummary()
expected_values = [('a', 'http://www.foo.com/', 3),
('b', 'http://www.foo.com/', 4),
('a', 'http://www.bar.com/', 1),
('d', 'http://www.bar.com/', 2)]
actual_values = [(v.name, v.page.url, v.value)
for v in results.all_page_specific_values]
self.assertEquals(expected_values, actual_values)
def test_url_is_invalid_value(self):
results = page_test_results.PageTestResults()
results.WillRunPage(self.pages[0])
self.assertRaises(
AssertionError,
lambda: results.AddValue(scalar.ScalarValue(
self.pages[0], 'url', 'string', 'foo')))
def test_basic_summary(self):
test_page_set = _MakePageSet()
measurement_results = page_test_results.PageTestResults()
measurement_results.WillRunPage(test_page_set.pages[0])
measurement_results.AddValue(
scalar.ScalarValue(test_page_set.pages[0], 'a', 'seconds', 3))
measurement_results.DidRunPage(test_page_set.pages[0])
measurement_results.WillRunPage(test_page_set.pages[1])
measurement_results.AddValue(
scalar.ScalarValue(test_page_set.pages[1], 'a', 'seconds', 7))
measurement_results.DidRunPage(test_page_set.pages[1])
formatter = buildbot_output_formatter.BuildbotOutputFormatter(
self._test_output_stream)
formatter.Format(measurement_results)
expected = [
'RESULT a: http___www.bar.com_= 7 seconds\n',
'RESULT a: http___www.foo.com_= 3 seconds\n',
'*RESULT a: a= [3,7] seconds\nAvg a: 5.000000seconds\n' +
'Sd a: 2.828427seconds\n',
'RESULT telemetry_page_measurement_results: ' +
'num_failed= 0 count\n',
'RESULT telemetry_page_measurement_results: ' +
'num_errored= 0 count\n'
]
self.assertEquals(expected, self._test_output_stream.output_data)
def testResults(self):
cycler = self.SetUpCycler([], True)
pages = [
FakePage('http://fakepage1.com'),
FakePage('http://fakepage2.com')
]
tab = FakeTab()
for i in range(2):
for page in pages:
results = page_test_results.PageTestResults()
results.WillRunPage(page)
cycler.WillNavigateToPage(page, tab)
cycler.ValidateAndMeasurePage(page, tab, results)
results.DidRunPage(page)
values = results.all_page_specific_values
self.assertEqual(4, len(values))
self.assertEqual(values[0].page, page)
chart_name = 'cold_times' if i == 0 else 'warm_times'
self.assertEqual(values[0].name,
'%s.page_load_time' % chart_name)
self.assertEqual(values[0].units, 'ms')
for value, expected in zip(values[1:],
['gpu', 'renderer', 'browser']):
self.assertEqual(value.page, page)
self.assertEqual(
value.name,
'cpu_utilization.cpu_utilization_%s' % expected)
self.assertEqual(value.units, '%')
cycler.DidNavigateToPage(page, tab)
def test_histogram(self):
results = page_test_results.PageTestResults()
results.WillRunPage(self._page_set[0])
results.AddValue(
histogram.HistogramValue(
self._page_set[0],
'a',
'',
raw_value_json=
'{"buckets": [{"low": 1, "high": 2, "count": 1}]}'))
results.DidRunPage(self._page_set[0])
results.WillRunPage(self._page_set[1])
results.AddValue(
histogram.HistogramValue(
self._page_set[1],
'a',
'',
raw_value_json=
'{"buckets": [{"low": 2, "high": 3, "count": 1}]}'))
results.DidRunPage(self._page_set[1])
self._formatter.Format(results)
self.assertEqual(self.output_header_row, ['page_name', 'a ()'])
self.assertEqual(self.output_data_rows,
[[self._page_set[0].display_name, '1.5'],
[self._page_set[1].display_name, '2.5']])
def testBasicSummaryPassAndFailPage(self):
"""If a page failed, only print summary for individual pages."""
page0 = self.pages[0]
page1 = self.pages[1]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3)
results.AddValue(v0)
v1 = failure.FailureValue.FromMessage(page0, 'message')
results.AddValue(v1)
results.DidRunPage(page0)
results.WillRunPage(page1)
v2 = scalar.ScalarValue(page1, 'a', 'seconds', 7)
results.AddValue(v2)
results.DidRunPage(page1)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
v0_list = list_of_scalar_values.ListOfScalarValues(
page0, 'a', 'seconds', [3])
v2_list = list_of_scalar_values.ListOfScalarValues(
page1, 'a', 'seconds', [7])
self.assertEquals(2, len(values))
self.assertIn(v0_list, values)
self.assertIn(v2_list, values)
def testAsDictBaseKeys(self):
results = page_test_results.PageTestResults()
d = json_output_formatter.ResultsAsDict(
results, self._formatter.benchmark_metadata)
self.assertEquals(d['format_version'], '0.2')
self.assertEquals(d['benchmark_name'], 'benchmark_name')
def test_overall_results_page_runs_twice(self):
test_page_set = _MakePageSet()
measurement_results = page_test_results.PageTestResults()
measurement_results.AddSummaryValue(
scalar.ScalarValue(None, 'a', 'seconds', 1))
measurement_results.WillRunPage(test_page_set.pages[0])
measurement_results.AddValue(
scalar.ScalarValue(test_page_set.pages[0], 'b', 'seconds', 2))
measurement_results.DidRunPage(test_page_set.pages[0])
measurement_results.WillRunPage(test_page_set.pages[0])
measurement_results.AddValue(
scalar.ScalarValue(test_page_set.pages[0], 'b', 'seconds', 3))
measurement_results.DidRunPage(test_page_set.pages[0])
formatter = buildbot_output_formatter.BuildbotOutputFormatter(
self._test_output_stream)
formatter.Format(measurement_results)
expected = [
'RESULT b: http___www.foo.com_= [2,3] seconds\n' +
'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n',
'*RESULT b: b= [2,3] seconds\n' +
'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n',
'*RESULT a: a= 1 seconds\n',
'RESULT telemetry_page_measurement_results: num_failed= 0 count\n',
'RESULT telemetry_page_measurement_results: num_errored= 0 count\n'
]
self.assertEquals(expected, self._test_output_stream.output_data)
def testCacheHandled(self):
cycler = self.SetUpCycler(
['--pageset-repeat=5', '--cold-load-percent=50'], True)
url_name = 'http://fakepage.com'
page = FakePage(url_name)
tab = FakeTab()
for i in range(5):
results = page_test_results.PageTestResults()
results.WillRunPage(page)
cycler.WillNavigateToPage(page, tab)
self.assertEqual(
max(0, i - 2), tab.clear_cache_calls,
'Iteration %d tab.clear_cache_calls %d' %
(i, tab.clear_cache_calls))
cycler.ValidateAndMeasurePage(page, tab, results)
results.DidRunPage(page)
values = results.all_page_specific_values
self.assertGreater(len(values), 2)
self.assertEqual(values[0].page, page)
chart_name = 'cold_times' if i == 0 or i > 2 else 'warm_times'
self.assertEqual(values[0].name, '%s.page_load_time' % chart_name)
self.assertEqual(values[0].units, 'ms')
cycler.DidNavigateToPage(page, tab)
def test_basic_summary_pass_and_fail_page(self):
"""If a page failed, only print summary for individual pages."""
test_page_set = _MakePageSet()
measurement_results = page_test_results.PageTestResults()
measurement_results.WillRunPage(test_page_set.pages[0])
measurement_results.AddValue(
scalar.ScalarValue(test_page_set.pages[0], 'a', 'seconds', 3))
measurement_results.AddValue(
failure.FailureValue.FromMessage(test_page_set.pages[0],
'message'))
measurement_results.DidRunPage(test_page_set.pages[0])
measurement_results.WillRunPage(test_page_set.pages[1])
measurement_results.AddValue(
scalar.ScalarValue(test_page_set.pages[1], 'a', 'seconds', 7))
measurement_results.DidRunPage(test_page_set.pages[1])
formatter = buildbot_output_formatter.BuildbotOutputFormatter(
self._test_output_stream)
formatter.Format(measurement_results)
expected = [
'RESULT a: http___www.bar.com_= 7 seconds\n',
'RESULT a: http___www.foo.com_= 3 seconds\n',
'RESULT telemetry_page_measurement_results: ' +
'num_failed= 1 count\n',
'RESULT telemetry_page_measurement_results: ' +
'num_errored= 0 count\n'
]
self.assertEquals(expected, self._test_output_stream.output_data)
def testBasicSummary(self):
page0 = self.pages[0]
page1 = self.pages[1]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3)
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page1)
v1 = scalar.ScalarValue(page1, 'a', 'seconds', 7)
results.AddValue(v1)
results.DidRunPage(page1)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
v0_list = list_of_scalar_values.ListOfScalarValues(
page0, 'a', 'seconds', [3])
v1_list = list_of_scalar_values.ListOfScalarValues(
page1, 'a', 'seconds', [7])
merged_value = list_of_scalar_values.ListOfScalarValues(
None, 'a', 'seconds', [3, 7])
self.assertEquals(3, len(values))
self.assertIn(v0_list, values)
self.assertIn(v1_list, values)
self.assertIn(merged_value, values)
def testColdWarm(self):
cycler = self.SetUpCycler(['--pageset-repeat=3'], True)
pages = [
FakePage('http://fakepage1.com'),
FakePage('http://fakepage2.com')
]
tab = FakeTab()
for i in range(3):
for page in pages:
results = page_test_results.PageTestResults()
results.WillRunPage(page)
cycler.WillNavigateToPage(page, tab)
cycler.ValidateAndMeasurePage(page, tab, results)
results.DidRunPage(page)
values = results.all_page_specific_values
self.assertGreater(len(values), 2)
self.assertEqual(values[0].page, page)
chart_name = 'cold_times' if i == 0 or i > 1 else 'warm_times'
self.assertEqual(values[0].name,
'%s.page_load_time' % chart_name)
self.assertEqual(values[0].units, 'ms')
cycler.DidNavigateToPage(page, tab)
def testHistogram(self):
page0 = self.pages[0]
page1 = self.pages[1]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = histogram.HistogramValue(
page0,
'a',
'units',
raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}',
important=False)
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page1)
v1 = histogram.HistogramValue(
page1,
'a',
'units',
raw_value_json='{"buckets": [{"low": 2, "high": 3, "count": 1}]}',
important=False)
results.AddValue(v1)
results.DidRunPage(page1)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
self.assertEquals(2, len(values))
self.assertIn(v0, values)
self.assertIn(v1, values)
def testListValue(self):
page0 = self.pages[0]
page1 = self.pages[1]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = list_of_scalar_values.ListOfScalarValues(page0, 'b', 'seconds',
[2, 2])
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page1)
v1 = list_of_scalar_values.ListOfScalarValues(page1, 'b', 'seconds',
[3, 3])
results.AddValue(v1)
results.DidRunPage(page1)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
b_summary = list_of_scalar_values.ListOfScalarValues(
None, 'b', 'seconds', [2, 2, 3, 3])
self.assertEquals(3, len(values))
self.assertIn(v0, values)
self.assertIn(v1, values)
self.assertIn(b_summary, values)
def testPageRunsTwice(self):
page0 = self.pages[0]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'b', 'seconds', 2)
results.AddValue(v0)
results.DidRunPage(page0)
results.WillRunPage(page0)
v1 = scalar.ScalarValue(page0, 'b', 'seconds', 3)
results.AddValue(v1)
results.DidRunPage(page0)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
page0_aggregated = list_of_scalar_values.ListOfScalarValues(
page0, 'b', 'seconds', [2, 3])
b_summary = list_of_scalar_values.ListOfScalarValues(
None, 'b', 'seconds', [2, 3])
self.assertEquals(2, len(values))
self.assertIn(page0_aggregated, values)
self.assertIn(b_summary, values)
def Main(args):
if len(args) is not 1:
print 'Invalid arguments. Usage: measure_trace.py <trace file>'
return 1
with open(args[0]) as trace_file:
trace_data = tracing_timeline_data.TracingTimelineData(
json.load(trace_file))
timeline_model = model.TimelineModel(trace_data)
smoothness_metric = smoothness.SmoothnessMetric()
formatters = [
buildbot_output_formatter.BuildbotOutputFormatter(sys.stdout)
]
results = page_test_results.PageTestResults(output_formatters=formatters)
for thread in timeline_model.GetAllThreads():
interaction_records = _ExtractInteractionsRecordFromThread(
thread, timeline_model)
if not any(interaction_records):
continue
records_label_to_records_map = collections.defaultdict(list)
for r in interaction_records:
records_label_to_records_map[r.label].append(r)
for label, records in records_label_to_records_map.iteritems():
if records[0].is_smooth:
page = page_module.Page('interaction-record://%s' % label)
results.WillRunPage(page)
smoothness_metric.AddResults(timeline_model, thread, records,
results)
results.DidRunPage(page)
results.PrintSummary()
return 0
def testStreamingResults(self):
test_page_set = _MakePageSet()
results = page_test_results.PageTestResults(
progress_reporter=self._reporter)
exc_info = self.CreateException()
results.WillRunPage(test_page_set.pages[0])
self._mock_timer.SetTime(0.007)
results.DidRunPage(test_page_set.pages[0])
expected = ('[ RUN ] http://www.foo.com/\n'
'[ OK ] http://www.foo.com/ (7 ms)\n')
self.assertEquals(expected, ''.join(self._output_stream.output_data))
results.WillRunPage(test_page_set.pages[1])
self._mock_timer.SetTime(0.009)
exception_trace = ''.join(traceback.format_exception(*exc_info))
results.AddValue(failure.FailureValue(test_page_set.pages[1],
exc_info))
results.DidRunPage(test_page_set.pages[1])
expected = ('[ RUN ] http://www.foo.com/\n'
'[ OK ] http://www.foo.com/ (7 ms)\n'
'[ RUN ] http://www.bar.com/\n'
'%s\n'
'[ FAILED ] http://www.bar.com/ (2 ms)\n' %
exception_trace)
def __init__(self):
self._model = model_module.TimelineModel()
renderer_process = self._model.GetOrCreateProcess(1)
self._renderer_thread = renderer_process.GetOrCreateThread(2)
self._renderer_thread.name = 'CrRendererMain'
self._results = page_test_results.PageTestResults()
self._metric = None
self._ps = None
def _MeasureFakePage(histograms):
results = page_test_results.PageTestResults()
page = FakePage('file://blank.html')
tab = FakeTab(histograms)
metric = v8_detached_context_age_in_gc.V8DetachedContextAgeInGC()
results.WillRunPage(page)
metric.DidNavigateToPage(page, tab)
metric.ValidateAndMeasurePage(page, tab, results)
results.DidRunPage(page)
return results
def testAsDictWithSummaryValueOnly(self):
results = page_test_results.PageTestResults()
v = scalar.ScalarValue(None, 'baz', 'seconds', 5)
results.AddSummaryValue(v)
d = json_output_formatter.ResultsAsDict(
results, self._formatter.benchmark_metadata)
self.assertFalse(d['pages'])
self.assertTrue(_HasValueNamed(d['summary_values'], 'baz'))
def test_unit_change(self):
results = page_test_results.PageTestResults()
results.WillRunPage(self.pages[0])
results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3))
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
self.assertRaises(
AssertionError, lambda: results.AddValue(
scalar.ScalarValue(self.pages[1], 'a', 'foobgrobbers', 3)))
def testBasicSummaryNonuniformResults(self):
page0 = self.pages[0]
page1 = self.pages[1]
page2 = self.pages[2]
results = page_test_results.PageTestResults()
results.WillRunPage(page0)
v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3)
results.AddValue(v0)
v1 = scalar.ScalarValue(page0, 'b', 'seconds', 10)
results.AddValue(v1)
results.DidRunPage(page0)
results.WillRunPage(page1)
v2 = scalar.ScalarValue(page1, 'a', 'seconds', 3)
results.AddValue(v2)
v3 = scalar.ScalarValue(page1, 'b', 'seconds', 10)
results.AddValue(v3)
results.DidRunPage(page1)
results.WillRunPage(page2)
v4 = scalar.ScalarValue(page2, 'a', 'seconds', 7)
results.AddValue(v4)
# Note, page[2] does not report a 'b' metric.
results.DidRunPage(page2)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
v0_list = list_of_scalar_values.ListOfScalarValues(
page0, 'a', 'seconds', [3])
v1_list = list_of_scalar_values.ListOfScalarValues(
page0, 'b', 'seconds', [10])
v2_list = list_of_scalar_values.ListOfScalarValues(
page1, 'a', 'seconds', [3])
v3_list = list_of_scalar_values.ListOfScalarValues(
page1, 'b', 'seconds', [10])
v4_list = list_of_scalar_values.ListOfScalarValues(
page2, 'a', 'seconds', [7])
a_summary = list_of_scalar_values.ListOfScalarValues(
None, 'a', 'seconds', [3, 3, 7])
b_summary = list_of_scalar_values.ListOfScalarValues(
None, 'b', 'seconds', [10, 10])
self.assertEquals(7, len(values))
self.assertIn(v0_list, values)
self.assertIn(v1_list, values)
self.assertIn(v2_list, values)
self.assertIn(v3_list, values)
self.assertIn(v4_list, values)
self.assertIn(a_summary, values)
self.assertIn(b_summary, values)
def testAsDictWithOnePage(self):
results = page_test_results.PageTestResults()
results.WillRunPage(self._page_set[0])
v0 = scalar.ScalarValue(results.current_page, 'foo', 'seconds', 3)
results.AddValue(v0)
results.DidRunPage(self._page_set[0])
d = json_output_formatter.ResultsAsDict(
results, self._formatter.benchmark_metadata)
self.assertTrue(_HasPage(d['pages'], self._page_set[0]))
self.assertTrue(_HasValueNamed(d['per_page_values'], 'foo'))
def test_type_change(self):
results = page_test_results.PageTestResults()
results.WillRunPage(self.pages[0])
results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3))
results.DidRunPage(self.pages[0])
results.WillRunPage(self.pages[1])
self.assertRaises(
AssertionError,
lambda: results.AddValue(histogram.HistogramValue(
self.pages[1], 'a', 'seconds',
raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}')))
def testOutputAndParse(self):
results = page_test_results.PageTestResults()
self._output.truncate(0)
results.WillRunPage(self._page_set[0])
v0 = scalar.ScalarValue(results.current_page, 'foo', 'seconds', 3)
results.AddValue(v0)
results.DidRunPage(self._page_set[0])
self._formatter.Format(results)
json.loads(self._output.getvalue())
def testPopulateResultsFromStats(self):
stats = _MockRenderingStats()
for stat in _MockRenderingStats.stats:
# Just set fake data for all of the relevant arrays of stats typically
# found in a RenderingStats object.
setattr(stats, stat, [[10, 20], [30, 40, 50]])
results = page_test_results.PageTestResults()
results.WillRunPage(self.page)
self.metric._PopulateResultsFromStats(results, stats, False)
current_page_run = results.current_page_run
self.assertTrue(current_page_run.ok)
self.assertEquals(11, len(current_page_run.values))
def MeasureFakePage(self, metric):
self._renderer_thread.async_slices.extend(self._async_slices)
self._model.FinalizeImport()
interaction_records = [
tir_module.TimelineInteractionRecord.FromAsyncEvent(s)
for s in self._async_slices]
results = page_test_results.PageTestResults()
fake_page = None
results.WillRunPage(fake_page)
metric.AddResults(self._model, self._renderer_thread, interaction_records,
results)
results.DidRunPage(fake_page)
return results
def testAvgStddev(self):
results = page_test_results.PageTestResults()
results.WillRunPage(page.Page('file://blank.html'))
events = map(FakeEvent._make, [(name, 42, 43) for name in
layout.LayoutMetric.EVENTS])
layout.LayoutMetric()._AddResults(events, results)
expected = set()
for name in layout.LayoutMetric.EVENTS.itervalues():
expected.add((name + '_avg', 1))
expected.add((name + '_stddev', 0))
actual = set((value.name, value.value) for value in
results.current_page_run.values)
self.assertEquals(expected, actual)