def Run(self, args): """Run this test with the given options.""" self.CustomizeBrowserOptions(args.browser_options) test = self.PageTestClass()() test.__name__ = self.__class__.__name__ if hasattr(self, '_disabled_strings'): test._disabled_strings = self._disabled_strings if hasattr(self, '_enabled_strings'): test._enabled_strings = self._enabled_strings ps = self.CreatePageSet(args) expectations = self.CreateExpectations(ps) self._DownloadGeneratedProfileArchive(args) results = page_test_results.PageTestResults() try: results = page_runner.Run(test, ps, expectations, args) except page_test.TestNotSupportedOnPlatformFailure as failure: logging.warning(str(failure)) results.PrintSummary() return len(results.failures) + len(results.errors)
def test_add_summary_value_with_page_specified(self): results = page_test_results.PageTestResults() results.WillRunPage(self.pages[0]) self.assertRaises( AssertionError, lambda: results.AddSummaryValue(scalar.ScalarValue(self.pages[0], 'a', 'units', 3)))
def testFilterIsFirstResult(self): def AcceptSecondValues(_, is_first_result): return not is_first_result results = page_test_results.PageTestResults( value_can_be_added_predicate=AcceptSecondValues) # First results (filtered out) results.WillRunPage(self.pages[0]) results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 7)) results.AddValue(scalar.ScalarValue(self.pages[0], 'b', 'seconds', 8)) results.DidRunPage(self.pages[0]) results.WillRunPage(self.pages[1]) results.AddValue(scalar.ScalarValue(self.pages[1], 'a', 'seconds', 5)) results.AddValue(scalar.ScalarValue(self.pages[1], 'd', 'seconds', 6)) results.DidRunPage(self.pages[1]) # Second results results.WillRunPage(self.pages[0]) results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) results.AddValue(scalar.ScalarValue(self.pages[0], 'b', 'seconds', 4)) results.DidRunPage(self.pages[0]) results.WillRunPage(self.pages[1]) results.AddValue(scalar.ScalarValue(self.pages[1], 'a', 'seconds', 1)) results.AddValue(scalar.ScalarValue(self.pages[1], 'd', 'seconds', 2)) results.DidRunPage(self.pages[1]) results.PrintSummary() expected_values = [('a', 'http://www.foo.com/', 3), ('b', 'http://www.foo.com/', 4), ('a', 'http://www.bar.com/', 1), ('d', 'http://www.bar.com/', 2)] actual_values = [(v.name, v.page.url, v.value) for v in results.all_page_specific_values] self.assertEquals(expected_values, actual_values)
def test_url_is_invalid_value(self): results = page_test_results.PageTestResults() results.WillRunPage(self.pages[0]) self.assertRaises( AssertionError, lambda: results.AddValue(scalar.ScalarValue( self.pages[0], 'url', 'string', 'foo')))
def test_basic_summary(self): test_page_set = _MakePageSet() measurement_results = page_test_results.PageTestResults() measurement_results.WillRunPage(test_page_set.pages[0]) measurement_results.AddValue( scalar.ScalarValue(test_page_set.pages[0], 'a', 'seconds', 3)) measurement_results.DidRunPage(test_page_set.pages[0]) measurement_results.WillRunPage(test_page_set.pages[1]) measurement_results.AddValue( scalar.ScalarValue(test_page_set.pages[1], 'a', 'seconds', 7)) measurement_results.DidRunPage(test_page_set.pages[1]) formatter = buildbot_output_formatter.BuildbotOutputFormatter( self._test_output_stream) formatter.Format(measurement_results) expected = [ 'RESULT a: http___www.bar.com_= 7 seconds\n', 'RESULT a: http___www.foo.com_= 3 seconds\n', '*RESULT a: a= [3,7] seconds\nAvg a: 5.000000seconds\n' + 'Sd a: 2.828427seconds\n', 'RESULT telemetry_page_measurement_results: ' + 'num_failed= 0 count\n', 'RESULT telemetry_page_measurement_results: ' + 'num_errored= 0 count\n' ] self.assertEquals(expected, self._test_output_stream.output_data)
def testResults(self): cycler = self.SetUpCycler([], True) pages = [ FakePage('http://fakepage1.com'), FakePage('http://fakepage2.com') ] tab = FakeTab() for i in range(2): for page in pages: results = page_test_results.PageTestResults() results.WillRunPage(page) cycler.WillNavigateToPage(page, tab) cycler.ValidateAndMeasurePage(page, tab, results) results.DidRunPage(page) values = results.all_page_specific_values self.assertEqual(4, len(values)) self.assertEqual(values[0].page, page) chart_name = 'cold_times' if i == 0 else 'warm_times' self.assertEqual(values[0].name, '%s.page_load_time' % chart_name) self.assertEqual(values[0].units, 'ms') for value, expected in zip(values[1:], ['gpu', 'renderer', 'browser']): self.assertEqual(value.page, page) self.assertEqual( value.name, 'cpu_utilization.cpu_utilization_%s' % expected) self.assertEqual(value.units, '%') cycler.DidNavigateToPage(page, tab)
def test_histogram(self): results = page_test_results.PageTestResults() results.WillRunPage(self._page_set[0]) results.AddValue( histogram.HistogramValue( self._page_set[0], 'a', '', raw_value_json= '{"buckets": [{"low": 1, "high": 2, "count": 1}]}')) results.DidRunPage(self._page_set[0]) results.WillRunPage(self._page_set[1]) results.AddValue( histogram.HistogramValue( self._page_set[1], 'a', '', raw_value_json= '{"buckets": [{"low": 2, "high": 3, "count": 1}]}')) results.DidRunPage(self._page_set[1]) self._formatter.Format(results) self.assertEqual(self.output_header_row, ['page_name', 'a ()']) self.assertEqual(self.output_data_rows, [[self._page_set[0].display_name, '1.5'], [self._page_set[1].display_name, '2.5']])
def testBasicSummaryPassAndFailPage(self): """If a page failed, only print summary for individual pages.""" page0 = self.pages[0] page1 = self.pages[1] results = page_test_results.PageTestResults() results.WillRunPage(page0) v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3) results.AddValue(v0) v1 = failure.FailureValue.FromMessage(page0, 'message') results.AddValue(v1) results.DidRunPage(page0) results.WillRunPage(page1) v2 = scalar.ScalarValue(page1, 'a', 'seconds', 7) results.AddValue(v2) results.DidRunPage(page1) summary = summary_module.Summary(results.all_page_specific_values) values = summary.interleaved_computed_per_page_values_and_summaries v0_list = list_of_scalar_values.ListOfScalarValues( page0, 'a', 'seconds', [3]) v2_list = list_of_scalar_values.ListOfScalarValues( page1, 'a', 'seconds', [7]) self.assertEquals(2, len(values)) self.assertIn(v0_list, values) self.assertIn(v2_list, values)
def testAsDictBaseKeys(self): results = page_test_results.PageTestResults() d = json_output_formatter.ResultsAsDict( results, self._formatter.benchmark_metadata) self.assertEquals(d['format_version'], '0.2') self.assertEquals(d['benchmark_name'], 'benchmark_name')
def test_overall_results_page_runs_twice(self): test_page_set = _MakePageSet() measurement_results = page_test_results.PageTestResults() measurement_results.AddSummaryValue( scalar.ScalarValue(None, 'a', 'seconds', 1)) measurement_results.WillRunPage(test_page_set.pages[0]) measurement_results.AddValue( scalar.ScalarValue(test_page_set.pages[0], 'b', 'seconds', 2)) measurement_results.DidRunPage(test_page_set.pages[0]) measurement_results.WillRunPage(test_page_set.pages[0]) measurement_results.AddValue( scalar.ScalarValue(test_page_set.pages[0], 'b', 'seconds', 3)) measurement_results.DidRunPage(test_page_set.pages[0]) formatter = buildbot_output_formatter.BuildbotOutputFormatter( self._test_output_stream) formatter.Format(measurement_results) expected = [ 'RESULT b: http___www.foo.com_= [2,3] seconds\n' + 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', '*RESULT b: b= [2,3] seconds\n' + 'Avg b: 2.500000seconds\nSd b: 0.707107seconds\n', '*RESULT a: a= 1 seconds\n', 'RESULT telemetry_page_measurement_results: num_failed= 0 count\n', 'RESULT telemetry_page_measurement_results: num_errored= 0 count\n' ] self.assertEquals(expected, self._test_output_stream.output_data)
def testCacheHandled(self): cycler = self.SetUpCycler( ['--pageset-repeat=5', '--cold-load-percent=50'], True) url_name = 'http://fakepage.com' page = FakePage(url_name) tab = FakeTab() for i in range(5): results = page_test_results.PageTestResults() results.WillRunPage(page) cycler.WillNavigateToPage(page, tab) self.assertEqual( max(0, i - 2), tab.clear_cache_calls, 'Iteration %d tab.clear_cache_calls %d' % (i, tab.clear_cache_calls)) cycler.ValidateAndMeasurePage(page, tab, results) results.DidRunPage(page) values = results.all_page_specific_values self.assertGreater(len(values), 2) self.assertEqual(values[0].page, page) chart_name = 'cold_times' if i == 0 or i > 2 else 'warm_times' self.assertEqual(values[0].name, '%s.page_load_time' % chart_name) self.assertEqual(values[0].units, 'ms') cycler.DidNavigateToPage(page, tab)
def test_basic_summary_pass_and_fail_page(self): """If a page failed, only print summary for individual pages.""" test_page_set = _MakePageSet() measurement_results = page_test_results.PageTestResults() measurement_results.WillRunPage(test_page_set.pages[0]) measurement_results.AddValue( scalar.ScalarValue(test_page_set.pages[0], 'a', 'seconds', 3)) measurement_results.AddValue( failure.FailureValue.FromMessage(test_page_set.pages[0], 'message')) measurement_results.DidRunPage(test_page_set.pages[0]) measurement_results.WillRunPage(test_page_set.pages[1]) measurement_results.AddValue( scalar.ScalarValue(test_page_set.pages[1], 'a', 'seconds', 7)) measurement_results.DidRunPage(test_page_set.pages[1]) formatter = buildbot_output_formatter.BuildbotOutputFormatter( self._test_output_stream) formatter.Format(measurement_results) expected = [ 'RESULT a: http___www.bar.com_= 7 seconds\n', 'RESULT a: http___www.foo.com_= 3 seconds\n', 'RESULT telemetry_page_measurement_results: ' + 'num_failed= 1 count\n', 'RESULT telemetry_page_measurement_results: ' + 'num_errored= 0 count\n' ] self.assertEquals(expected, self._test_output_stream.output_data)
def testBasicSummary(self): page0 = self.pages[0] page1 = self.pages[1] results = page_test_results.PageTestResults() results.WillRunPage(page0) v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3) results.AddValue(v0) results.DidRunPage(page0) results.WillRunPage(page1) v1 = scalar.ScalarValue(page1, 'a', 'seconds', 7) results.AddValue(v1) results.DidRunPage(page1) summary = summary_module.Summary(results.all_page_specific_values) values = summary.interleaved_computed_per_page_values_and_summaries v0_list = list_of_scalar_values.ListOfScalarValues( page0, 'a', 'seconds', [3]) v1_list = list_of_scalar_values.ListOfScalarValues( page1, 'a', 'seconds', [7]) merged_value = list_of_scalar_values.ListOfScalarValues( None, 'a', 'seconds', [3, 7]) self.assertEquals(3, len(values)) self.assertIn(v0_list, values) self.assertIn(v1_list, values) self.assertIn(merged_value, values)
def testColdWarm(self): cycler = self.SetUpCycler(['--pageset-repeat=3'], True) pages = [ FakePage('http://fakepage1.com'), FakePage('http://fakepage2.com') ] tab = FakeTab() for i in range(3): for page in pages: results = page_test_results.PageTestResults() results.WillRunPage(page) cycler.WillNavigateToPage(page, tab) cycler.ValidateAndMeasurePage(page, tab, results) results.DidRunPage(page) values = results.all_page_specific_values self.assertGreater(len(values), 2) self.assertEqual(values[0].page, page) chart_name = 'cold_times' if i == 0 or i > 1 else 'warm_times' self.assertEqual(values[0].name, '%s.page_load_time' % chart_name) self.assertEqual(values[0].units, 'ms') cycler.DidNavigateToPage(page, tab)
def testHistogram(self): page0 = self.pages[0] page1 = self.pages[1] results = page_test_results.PageTestResults() results.WillRunPage(page0) v0 = histogram.HistogramValue( page0, 'a', 'units', raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}', important=False) results.AddValue(v0) results.DidRunPage(page0) results.WillRunPage(page1) v1 = histogram.HistogramValue( page1, 'a', 'units', raw_value_json='{"buckets": [{"low": 2, "high": 3, "count": 1}]}', important=False) results.AddValue(v1) results.DidRunPage(page1) summary = summary_module.Summary(results.all_page_specific_values) values = summary.interleaved_computed_per_page_values_and_summaries self.assertEquals(2, len(values)) self.assertIn(v0, values) self.assertIn(v1, values)
def testListValue(self): page0 = self.pages[0] page1 = self.pages[1] results = page_test_results.PageTestResults() results.WillRunPage(page0) v0 = list_of_scalar_values.ListOfScalarValues(page0, 'b', 'seconds', [2, 2]) results.AddValue(v0) results.DidRunPage(page0) results.WillRunPage(page1) v1 = list_of_scalar_values.ListOfScalarValues(page1, 'b', 'seconds', [3, 3]) results.AddValue(v1) results.DidRunPage(page1) summary = summary_module.Summary(results.all_page_specific_values) values = summary.interleaved_computed_per_page_values_and_summaries b_summary = list_of_scalar_values.ListOfScalarValues( None, 'b', 'seconds', [2, 2, 3, 3]) self.assertEquals(3, len(values)) self.assertIn(v0, values) self.assertIn(v1, values) self.assertIn(b_summary, values)
def testPageRunsTwice(self): page0 = self.pages[0] results = page_test_results.PageTestResults() results.WillRunPage(page0) v0 = scalar.ScalarValue(page0, 'b', 'seconds', 2) results.AddValue(v0) results.DidRunPage(page0) results.WillRunPage(page0) v1 = scalar.ScalarValue(page0, 'b', 'seconds', 3) results.AddValue(v1) results.DidRunPage(page0) summary = summary_module.Summary(results.all_page_specific_values) values = summary.interleaved_computed_per_page_values_and_summaries page0_aggregated = list_of_scalar_values.ListOfScalarValues( page0, 'b', 'seconds', [2, 3]) b_summary = list_of_scalar_values.ListOfScalarValues( None, 'b', 'seconds', [2, 3]) self.assertEquals(2, len(values)) self.assertIn(page0_aggregated, values) self.assertIn(b_summary, values)
def Main(args): if len(args) is not 1: print 'Invalid arguments. Usage: measure_trace.py <trace file>' return 1 with open(args[0]) as trace_file: trace_data = tracing_timeline_data.TracingTimelineData( json.load(trace_file)) timeline_model = model.TimelineModel(trace_data) smoothness_metric = smoothness.SmoothnessMetric() formatters = [ buildbot_output_formatter.BuildbotOutputFormatter(sys.stdout) ] results = page_test_results.PageTestResults(output_formatters=formatters) for thread in timeline_model.GetAllThreads(): interaction_records = _ExtractInteractionsRecordFromThread( thread, timeline_model) if not any(interaction_records): continue records_label_to_records_map = collections.defaultdict(list) for r in interaction_records: records_label_to_records_map[r.label].append(r) for label, records in records_label_to_records_map.iteritems(): if records[0].is_smooth: page = page_module.Page('interaction-record://%s' % label) results.WillRunPage(page) smoothness_metric.AddResults(timeline_model, thread, records, results) results.DidRunPage(page) results.PrintSummary() return 0
def testStreamingResults(self): test_page_set = _MakePageSet() results = page_test_results.PageTestResults( progress_reporter=self._reporter) exc_info = self.CreateException() results.WillRunPage(test_page_set.pages[0]) self._mock_timer.SetTime(0.007) results.DidRunPage(test_page_set.pages[0]) expected = ('[ RUN ] http://www.foo.com/\n' '[ OK ] http://www.foo.com/ (7 ms)\n') self.assertEquals(expected, ''.join(self._output_stream.output_data)) results.WillRunPage(test_page_set.pages[1]) self._mock_timer.SetTime(0.009) exception_trace = ''.join(traceback.format_exception(*exc_info)) results.AddValue(failure.FailureValue(test_page_set.pages[1], exc_info)) results.DidRunPage(test_page_set.pages[1]) expected = ('[ RUN ] http://www.foo.com/\n' '[ OK ] http://www.foo.com/ (7 ms)\n' '[ RUN ] http://www.bar.com/\n' '%s\n' '[ FAILED ] http://www.bar.com/ (2 ms)\n' % exception_trace)
def __init__(self): self._model = model_module.TimelineModel() renderer_process = self._model.GetOrCreateProcess(1) self._renderer_thread = renderer_process.GetOrCreateThread(2) self._renderer_thread.name = 'CrRendererMain' self._results = page_test_results.PageTestResults() self._metric = None self._ps = None
def _MeasureFakePage(histograms): results = page_test_results.PageTestResults() page = FakePage('file://blank.html') tab = FakeTab(histograms) metric = v8_detached_context_age_in_gc.V8DetachedContextAgeInGC() results.WillRunPage(page) metric.DidNavigateToPage(page, tab) metric.ValidateAndMeasurePage(page, tab, results) results.DidRunPage(page) return results
def testAsDictWithSummaryValueOnly(self): results = page_test_results.PageTestResults() v = scalar.ScalarValue(None, 'baz', 'seconds', 5) results.AddSummaryValue(v) d = json_output_formatter.ResultsAsDict( results, self._formatter.benchmark_metadata) self.assertFalse(d['pages']) self.assertTrue(_HasValueNamed(d['summary_values'], 'baz'))
def test_unit_change(self): results = page_test_results.PageTestResults() results.WillRunPage(self.pages[0]) results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) results.DidRunPage(self.pages[0]) results.WillRunPage(self.pages[1]) self.assertRaises( AssertionError, lambda: results.AddValue( scalar.ScalarValue(self.pages[1], 'a', 'foobgrobbers', 3)))
def testBasicSummaryNonuniformResults(self): page0 = self.pages[0] page1 = self.pages[1] page2 = self.pages[2] results = page_test_results.PageTestResults() results.WillRunPage(page0) v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3) results.AddValue(v0) v1 = scalar.ScalarValue(page0, 'b', 'seconds', 10) results.AddValue(v1) results.DidRunPage(page0) results.WillRunPage(page1) v2 = scalar.ScalarValue(page1, 'a', 'seconds', 3) results.AddValue(v2) v3 = scalar.ScalarValue(page1, 'b', 'seconds', 10) results.AddValue(v3) results.DidRunPage(page1) results.WillRunPage(page2) v4 = scalar.ScalarValue(page2, 'a', 'seconds', 7) results.AddValue(v4) # Note, page[2] does not report a 'b' metric. results.DidRunPage(page2) summary = summary_module.Summary(results.all_page_specific_values) values = summary.interleaved_computed_per_page_values_and_summaries v0_list = list_of_scalar_values.ListOfScalarValues( page0, 'a', 'seconds', [3]) v1_list = list_of_scalar_values.ListOfScalarValues( page0, 'b', 'seconds', [10]) v2_list = list_of_scalar_values.ListOfScalarValues( page1, 'a', 'seconds', [3]) v3_list = list_of_scalar_values.ListOfScalarValues( page1, 'b', 'seconds', [10]) v4_list = list_of_scalar_values.ListOfScalarValues( page2, 'a', 'seconds', [7]) a_summary = list_of_scalar_values.ListOfScalarValues( None, 'a', 'seconds', [3, 3, 7]) b_summary = list_of_scalar_values.ListOfScalarValues( None, 'b', 'seconds', [10, 10]) self.assertEquals(7, len(values)) self.assertIn(v0_list, values) self.assertIn(v1_list, values) self.assertIn(v2_list, values) self.assertIn(v3_list, values) self.assertIn(v4_list, values) self.assertIn(a_summary, values) self.assertIn(b_summary, values)
def testAsDictWithOnePage(self): results = page_test_results.PageTestResults() results.WillRunPage(self._page_set[0]) v0 = scalar.ScalarValue(results.current_page, 'foo', 'seconds', 3) results.AddValue(v0) results.DidRunPage(self._page_set[0]) d = json_output_formatter.ResultsAsDict( results, self._formatter.benchmark_metadata) self.assertTrue(_HasPage(d['pages'], self._page_set[0])) self.assertTrue(_HasValueNamed(d['per_page_values'], 'foo'))
def test_type_change(self): results = page_test_results.PageTestResults() results.WillRunPage(self.pages[0]) results.AddValue(scalar.ScalarValue(self.pages[0], 'a', 'seconds', 3)) results.DidRunPage(self.pages[0]) results.WillRunPage(self.pages[1]) self.assertRaises( AssertionError, lambda: results.AddValue(histogram.HistogramValue( self.pages[1], 'a', 'seconds', raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}')))
def testOutputAndParse(self): results = page_test_results.PageTestResults() self._output.truncate(0) results.WillRunPage(self._page_set[0]) v0 = scalar.ScalarValue(results.current_page, 'foo', 'seconds', 3) results.AddValue(v0) results.DidRunPage(self._page_set[0]) self._formatter.Format(results) json.loads(self._output.getvalue())
def testPopulateResultsFromStats(self): stats = _MockRenderingStats() for stat in _MockRenderingStats.stats: # Just set fake data for all of the relevant arrays of stats typically # found in a RenderingStats object. setattr(stats, stat, [[10, 20], [30, 40, 50]]) results = page_test_results.PageTestResults() results.WillRunPage(self.page) self.metric._PopulateResultsFromStats(results, stats, False) current_page_run = results.current_page_run self.assertTrue(current_page_run.ok) self.assertEquals(11, len(current_page_run.values))
def MeasureFakePage(self, metric): self._renderer_thread.async_slices.extend(self._async_slices) self._model.FinalizeImport() interaction_records = [ tir_module.TimelineInteractionRecord.FromAsyncEvent(s) for s in self._async_slices] results = page_test_results.PageTestResults() fake_page = None results.WillRunPage(fake_page) metric.AddResults(self._model, self._renderer_thread, interaction_records, results) results.DidRunPage(fake_page) return results
def testAvgStddev(self): results = page_test_results.PageTestResults() results.WillRunPage(page.Page('file://blank.html')) events = map(FakeEvent._make, [(name, 42, 43) for name in layout.LayoutMetric.EVENTS]) layout.LayoutMetric()._AddResults(events, results) expected = set() for name in layout.LayoutMetric.EVENTS.itervalues(): expected.add((name + '_avg', 1)) expected.add((name + '_stddev', 0)) actual = set((value.name, value.value) for value in results.current_page_run.values) self.assertEquals(expected, actual)