def testTraceCaptureUponFailure(self): ps = self.CreateEmptyPageSet() ps.AddStory(FailedTimelinebasedMeasurementPage(ps, ps.base_dir)) options = tbm_module.Options() options.config.enable_chrome_trace = True options.SetTimelineBasedMetrics(['sampleMetric']) tbm = tbm_module.TimelineBasedMeasurement(options) results = self.RunMeasurement(tbm, ps, self._options) self.assertTrue(results.had_failures) self.assertEquals(1, len(results.FindAllTraceValues()))
def testSmoothnessTimelineBasedMeasurementForSmoke(self): ps = self.CreatePageSetFromFileInUnittestDataDir( 'interaction_enabled_page.html') setattr(ps.pages[0], 'RunSmoothness', { 'action': 'wait', 'javascript': 'window.animationDone' }) measurement = tbm_module.TimelineBasedMeasurement() results = self.RunMeasurement(measurement, ps, options=self._options) self.assertEquals(0, len(results.failures)) v = results.FindAllPageSpecificValuesNamed('CenterAnimation-jank') self.assertEquals(len(v), 1) v = results.FindAllPageSpecificValuesNamed('DrawerAnimation-jank') self.assertEquals(len(v), 1)
def testTimelineBasedMeasurementGestureAdjustmentSmoke(self): ps = self.CreateEmptyPageSet() ps.AddStory(TestTimelinebasedMeasurementPage( ps, ps.base_dir, trigger_scroll_gesture=True)) options = tbm_module.Options() options.SetLegacyTimelineBasedMetrics([smoothness.SmoothnessMetric()]) tbm = tbm_module.TimelineBasedMeasurement(options) results = self.RunMeasurement(tbm, ps, options=self._options) self.assertEquals(0, len(results.failures)) v = results.FindAllPageSpecificValuesFromIRNamed( 'Gesture_Scroll', 'frame_time_discrepancy') self.assertEquals(len(v), 1)
def testTimelineBasedMeasurementGestureAdjustmentSmoke(self): ps = self.CreateEmptyPageSet() ps.AddUserStory( TestTimelinebasedMeasurementPage(ps, ps.base_dir, trigger_scroll_gesture=True)) tbm = tbm_module.TimelineBasedMeasurement(tbm_module.Options()) measurement = tbpt_module.TimelineBasedPageTest(tbm) results = self.RunMeasurement(measurement, ps, options=self._options) self.assertEquals(0, len(results.failures)) v = results.FindAllPageSpecificValuesNamed( 'Gesture_Scroll-frame_time_discrepancy') self.assertEquals(len(v), 1)
def WillNavigateToPage(self, page, tab): tracing_controller = tab.browser.platform.tracing_controller # FIXME: Remove webkit.console when blink.console lands in chromium and # the ref builds are updated. crbug.com/386847 custom_categories = [ 'webkit.console', 'blink.console', 'benchmark', 'trace_event_overhead' ] category_filter = tracing_category_filter.TracingCategoryFilter( ','.join(custom_categories)) self._tbm = timeline_based_measurement.TimelineBasedMeasurement( timeline_based_measurement.Options(category_filter), _CustomResultsWrapper) self._tbm.WillRunUserStory(tracing_controller, page.GetSyntheticDelayCategories())
def testSmoothnessTimelineBasedMeasurementForSmoke(self): ps = self.CreateEmptyPageSet() ps.AddStory(TestTimelinebasedMeasurementPage( ps, ps.base_dir, trigger_animation=True)) tbm = tbm_module.TimelineBasedMeasurement(tbm_module.Options()) results = self.RunMeasurement(tbm, ps, options=self._options) self.assertEquals(0, len(results.failures)) v = results.FindAllPageSpecificValuesFromIRNamed( 'CenterAnimation', 'frame_time_discrepancy') self.assertEquals(len(v), 1) v = results.FindAllPageSpecificValuesFromIRNamed( 'DrawerAnimation', 'frame_time_discrepancy') self.assertEquals(len(v), 1)
def testSmoothnessTimelineBasedMeasurementForSmoke(self): ps = self.CreateEmptyPageSet() ps.AddPage( TestTimelinebasedMeasurementPage(ps, ps.base_dir, trigger_animation=True)) measurement = tbm_module.TimelineBasedMeasurement() results = self.RunMeasurement(measurement, ps, options=self._options) self.assertEquals(0, len(results.failures)) v = results.FindAllPageSpecificValuesNamed('CenterAnimation-jank') self.assertEquals(len(v), 1) v = results.FindAllPageSpecificValuesNamed('DrawerAnimation-jank') self.assertEquals(len(v), 1)
def WillNavigateToPage(self, page, tab): # FIXME: Remove webkit.console when blink.console lands in chromium and # the ref builds are updated. crbug.com/386847 custom_categories = [ 'webkit.console', 'blink.console', 'benchmark', 'trace_event_overhead' ] category_filter = chrome_trace_category_filter.ChromeTraceCategoryFilter( ','.join(custom_categories)) options = timeline_based_measurement.Options(category_filter) options.config.enable_platform_display_trace = True options.SetLegacyTimelineBasedMetrics([smoothness.SmoothnessMetric()]) self._tbm = timeline_based_measurement.TimelineBasedMeasurement( options, self._results_wrapper) self._tbm.WillRunStory(tab.browser.platform)
def testTraceCaptureUponFailure(self): ps = self.CreateEmptyPageSet() ps.AddStory(FailedTimelinebasedMeasurementPage(ps, ps.base_dir)) options = tbm_module.Options() options.config.enable_chrome_trace = True options.SetTimelineBasedMetrics(['sampleMetric']) tbm = tbm_module.TimelineBasedMeasurement(options) with tempfile_ext.NamedTemporaryDirectory() as tempdir: self._options.output_dir = tempdir results = self.RunMeasurement(tbm, ps, self._options) self.assertTrue(results.had_failures) runs = list(results.IterRunsWithTraces()) self.assertEquals(1, len(runs))
def testTraceCaptureUponFailure(self): story_set = self.CreateEmptyPageSet() story_set.AddStory( FailedTimelinebasedMeasurementPage(story_set, story_set.base_dir)) options = tbm_module.Options() options.config.enable_chrome_trace = True options.SetTimelineBasedMetrics(['sampleMetric']) tbm = tbm_module.TimelineBasedMeasurement(options) results = self.RunMeasurement(tbm, story_set, run_options=self._options) self.assertTrue(results.had_failures) runs = list(results.IterRunsWithTraces()) self.assertEquals(1, len(runs))
def testTBM2ForSmoke(self): ps = self.CreateEmptyPageSet() ps.AddStory(TestTimelinebasedMeasurementPage(ps, ps.base_dir)) options = tbm_module.Options() options.SetTimelineBasedMetrics(['sample_metric.html']) tbm = tbm_module.TimelineBasedMeasurement(options) results = self.RunMeasurement(tbm, ps, self._options) self.assertEquals(0, len(results.failures)) v_foo = results.FindAllPageSpecificValuesNamed('foo') v_bar = results.FindAllPageSpecificValuesNamed('bar') self.assertEquals(len(v_foo), 1) self.assertEquals(len(v_bar), 1) self.assertEquals(v_foo[0].value, 1) self.assertEquals(v_bar[0].value, 2)
def WillNavigateToPage(self, page, tab): # FIXME: Remove webkit.console when blink.console lands in chromium and # the ref builds are updated. crbug.com/386847 custom_categories = [ 'webkit.console', 'blink.console', 'benchmark', 'trace_event_overhead' ] category_filter = tracing_category_filter.TracingCategoryFilter( ','.join(custom_categories)) options = timeline_based_measurement.Options(category_filter) options.SetLegacyTimelineBasedMetrics([smoothness.SmoothnessMetric()]) for delay in page.GetSyntheticDelayCategories(): options.category_filter.AddSyntheticDelay(delay) self._tbm = timeline_based_measurement.TimelineBasedMeasurement( options, self._results_wrapper) self._tbm.WillRunStory(tab.browser.platform)
def testSuccessfulTimelineBasedMeasurementTest(self): """Check that PageTest is not required for story_runner.Run. Any PageTest related calls or attributes need to only be called for PageTest tests. """ class TestSharedTbmState(TestSharedState): def RunStory(self, results): pass test = timeline_based_measurement.TimelineBasedMeasurement( timeline_based_measurement.Options()) story_set = story_module.StorySet() story_set.AddStory(DummyLocalStory(TestSharedTbmState)) story_set.AddStory(DummyLocalStory(TestSharedTbmState)) story_set.AddStory(DummyLocalStory(TestSharedTbmState)) story_runner.Run(test, story_set, self.options, self.results) self.assertEquals(0, len(self.results.failures)) self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results))
def testTBM2ForSmoke(self): ps = self.CreateEmptyPageSet() ps.AddStory( TestTimelinebasedMeasurementPage( ps, ps.base_dir, additional_metrics=['sampleMetric'])) options = tbm_module.Options() options.config.enable_chrome_trace = True tbm = tbm_module.TimelineBasedMeasurement(options) results = self.RunMeasurement(tbm, ps, self._options) self.assertFalse(results.had_failures) tbm_metrics = [] for story_run in results.IterStoryRuns(): tbm_metrics += story_run.tbm_metrics self.assertEqual(tbm_metrics, ['sampleMetric']) histogram_dicts = results.AsHistogramDicts() hs = histogram_set.HistogramSet() hs.ImportDicts(histogram_dicts) self.assertEquals(4, len(hs)) hist = hs.GetFirstHistogram() benchmarks = hist.diagnostics.get(reserved_infos.BENCHMARKS.name) self.assertIsInstance(benchmarks, generic_set.GenericSet) self.assertEquals(1, len(benchmarks)) self.assertEquals(page_test_test_case.BENCHMARK_NAME, list(benchmarks)[0]) stories = hist.diagnostics.get(reserved_infos.STORIES.name) self.assertIsInstance(stories, generic_set.GenericSet) self.assertEquals(1, len(stories)) self.assertEquals('interaction_enabled_page.html', list(stories)[0]) repeats = hist.diagnostics.get(reserved_infos.STORYSET_REPEATS.name) self.assertIsInstance(repeats, generic_set.GenericSet) self.assertEquals(1, len(repeats)) self.assertEquals(0, list(repeats)[0]) hist = hs.GetFirstHistogram() trace_start = hist.diagnostics.get(reserved_infos.TRACE_START.name) self.assertIsInstance(trace_start, date_range.DateRange) v_foo = results.FindAllPageSpecificValuesNamed('foo_avg') self.assertEquals(len(v_foo), 1) self.assertEquals(v_foo[0].value, 50) self.assertIsNotNone(v_foo[0].page)
def testGPUTimesTimelineBasedMeasurementForSmoke(self): ps = self.CreateEmptyPageSet() ps.AddStory(TestTimelinebasedMeasurementPage( ps, ps.base_dir, trigger_animation=True)) cat_filter = tracing_category_filter.TracingCategoryFilter( 'disabled-by-default-gpu.service') tbm_option = tbm_module.Options(overhead_level=cat_filter) tbm = tbm_module.TimelineBasedMeasurement(tbm_option) results = self.RunMeasurement(tbm, ps, options=self._options) self.assertEquals(0, len(results.failures)) v = results.FindAllPageSpecificValuesFromIRNamed( 'CenterAnimation', 'browser_compositor_max_cpu_time') self.assertEquals(len(v), 1) self.assertGreater(v[0].value, 0) v = results.FindAllPageSpecificValuesFromIRNamed( 'DrawerAnimation', 'browser_compositor_max_cpu_time') self.assertEquals(len(v), 1) self.assertGreater(v[0].value, 0)
def testSuccessfulTimelineBasedMeasurementTest(self): """Check that PageTest is not required for story_runner.Run. Any PageTest related calls or attributes need to only be called for PageTest tests. """ class TestSharedTbmState(TestSharedState): def RunStory(self, results): pass TEST_WILL_RUN_STORY = 'test.WillRunStory' TEST_MEASURE = 'test.Measure' TEST_DID_RUN_STORY = 'test.DidRunStory' EXPECTED_CALLS_IN_ORDER = [TEST_WILL_RUN_STORY, TEST_MEASURE, TEST_DID_RUN_STORY] test = timeline_based_measurement.TimelineBasedMeasurement( timeline_based_measurement.Options()) manager = mock.MagicMock() test.WillRunStory = mock.MagicMock() test.Measure = mock.MagicMock() test.DidRunStory = mock.MagicMock() manager.attach_mock(test.WillRunStory, TEST_WILL_RUN_STORY) manager.attach_mock(test.Measure, TEST_MEASURE) manager.attach_mock(test.DidRunStory, TEST_DID_RUN_STORY) story_set = story_module.StorySet() story_set.AddStory(DummyLocalStory(TestSharedTbmState, name='foo')) story_set.AddStory(DummyLocalStory(TestSharedTbmState, name='bar')) story_set.AddStory(DummyLocalStory(TestSharedTbmState, name='baz')) story_runner.Run( test, story_set, self.options, self.results, metadata=EmptyMetadataForTest()) self.assertEquals(0, len(self.results.failures)) self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results)) self.assertEquals(3*EXPECTED_CALLS_IN_ORDER, [call[0] for call in manager.mock_calls])
def testMainthreadJankTimelineBasedMeasurement(self): ps = self.CreateEmptyPageSet() ps.AddStory(TestTimelinebasedMeasurementPage( ps, ps.base_dir, trigger_jank=True)) tbm = tbm_module.TimelineBasedMeasurement(tbm_module.Options()) results = self.RunMeasurement(tbm, ps, options=self._options) self.assertEquals(0, len(results.failures)) # In interaction_enabled_page.html, we create a jank loop based on # window.performance.now() (basically loop for x milliseconds). # Since window.performance.now() uses wall-time instead of thread time, # we only assert the biggest jank > 50ms here to account for the fact # that the browser may deschedule during the jank loop. v = results.FindAllPageSpecificValuesFromIRNamed( 'JankThreadJSRun', 'responsive-biggest_jank_thread_time') self.assertGreaterEqual(v[0].value, 50) v = results.FindAllPageSpecificValuesFromIRNamed( 'JankThreadJSRun', 'responsive-total_big_jank_thread_time') self.assertGreaterEqual(v[0].value, 50)
def testMainthreadJankTimelineBasedMeasurement(self): ps = self.CreateEmptyPageSet() ps.AddPage(TestTimelinebasedMeasurementPage(ps, ps.base_dir)) measurement = tbm_module.TimelineBasedMeasurement() results = self.RunMeasurement(measurement, ps, options=self._options) self.assertEquals(0, len(results.failures)) # In interaction_enabled_page.html, we create a jank loop based on # window.performance.now() (basically loop for x milliseconds). # Since window.performance.now() uses wall-time # instead of thread time, we set time to looping to 100ms in # interaction_enabled_page.html and only assert the biggest jank > 50ms here # to account for the fact that the browser may deschedule during the jank # loop. v = results.FindAllPageSpecificValuesNamed( 'JankThreadJSRun-responsive-biggest_jank_thread_time') self.assertGreaterEqual(v[0].value, 50) v = results.FindAllPageSpecificValuesNamed( 'JankThreadJSRun-responsive-total_big_jank_thread_time') self.assertGreaterEqual(v[0].value, 50)
def testTBM2ForSmoke(self): ps = self.CreateEmptyPageSet() ps.AddStory(TestTimelinebasedMeasurementPage(ps, ps.base_dir)) options = tbm_module.Options() options.config.enable_chrome_trace = True options.SetTimelineBasedMetric('sampleMetric') tbm = tbm_module.TimelineBasedMeasurement(options) results = self.RunMeasurement(tbm, ps, self._options) self.assertEquals(0, len(results.failures)) self.assertEquals(9, len(results.value_set)) self.assertEquals(1, len(results.value_set[0]['diagnostics'])) iter_info = results.value_set[0]['diagnostics']['iteration'] self.assertEqual('IterationInfo', iter_info['type']) self.assertEqual('', iter_info['benchmarkName']) self.assertEqual('interaction_enabled_page.html', iter_info['storyDisplayName']) self.assertEqual({}, iter_info['storyGroupingKeys']) self.assertEqual(0, iter_info['storyRepeatCounter']) self.assertEqual(0, iter_info['storysetRepeatCounter']) self.assertEqual('file://interaction_enabled_page.html', iter_info['storyUrl']) v_foo = results.FindAllPageSpecificValuesNamed('foo') v_bar = results.FindAllPageSpecificValuesNamed('bar') v_baz_avg = results.FindAllPageSpecificValuesNamed('baz_avg') v_baz_sum = results.FindAllPageSpecificValuesNamed('baz_sum') v_baz_count = results.FindAllPageSpecificValuesNamed('baz_count') self.assertEquals(len(v_foo), 1) self.assertEquals(len(v_bar), 1) self.assertEquals(v_foo[0].value, 1) self.assertIsNotNone(v_foo[0].page) self.assertEquals(v_bar[0].value, 2) self.assertIsNotNone(v_bar[0].page) self.assertEquals(len(v_baz_avg), 1) self.assertEquals(len(v_baz_sum), 1) self.assertEquals(len(v_baz_count), 1)
def testFastTimelineBasedMeasurementForSmoke(self): ps = self.CreateEmptyPageSet() ps.AddUserStory( TestTimelinebasedMeasurementPage(ps, ps.base_dir, trigger_slow=True)) tbm = tbm_module.TimelineBasedMeasurement(tbm_module.Options()) measurement = tbm_module.TimelineBasedPageTest(tbm) results = self.RunMeasurement(measurement, ps, options=self._options) self.assertEquals([], results.failures) expected_names = set([ 'SlowThreadJsRun-fast-duration', 'SlowThreadJsRun-fast-idle_time', 'SlowThreadJsRun-fast-incremental_marking', 'SlowThreadJsRun-fast-incremental_marking_outside_idle', 'SlowThreadJsRun-fast-mark_compactor', 'SlowThreadJsRun-fast-mark_compactor_outside_idle', 'SlowThreadJsRun-fast-scavenger', 'SlowThreadJsRun-fast-scavenger_outside_idle', 'SlowThreadJsRun-fast-total_garbage_collection', 'SlowThreadJsRun-fast-total_garbage_collection_outside_idle', 'trace', ]) if platform.GetHostPlatform().GetOSName() != 'win': # CPU metric is only supported non-Windows platforms. expected_names.add('SlowThreadJsRun-fast-cpu_time') self.assertEquals( expected_names, set(v.name for v in results.all_page_specific_values)) # In interaction_enabled_page.html, the "slow" interaction executes # a loop with window.performance.now() to wait 200ms. # fast-duration measures wall time so its value should be at least 200ms. v = results.FindAllPageSpecificValuesNamed( 'SlowThreadJsRun-fast-duration') self.assertGreaterEqual(v[0].value, 200.0)
def testTBM2ForSmoke(self): ps = self.CreateEmptyPageSet() ps.AddStory(TestTimelinebasedMeasurementPage(ps, ps.base_dir)) options = tbm_module.Options() options.config.enable_chrome_trace = True options.SetTimelineBasedMetrics(['sampleMetric']) tbm = tbm_module.TimelineBasedMeasurement(options) results = self.RunMeasurement(tbm, ps, self._options) self.assertEquals(0, len(results.failures)) self.assertEquals(1, len(results.histograms)) foos = results.histograms.GetHistogramsNamed('foo') self.assertEquals(1, len(foos)) hist = foos[0] benchmarks = hist.diagnostics.get(reserved_infos.BENCHMARKS.name) self.assertIsInstance(benchmarks, histogram.GenericSet) self.assertEquals(1, len(benchmarks)) self.assertEquals('', list(benchmarks)[0]) stories = hist.diagnostics.get(reserved_infos.STORIES.name) self.assertIsInstance(stories, histogram.GenericSet) self.assertEquals(1, len(stories)) self.assertEquals('interaction_enabled_page.html', list(stories)[0]) repeats = hist.diagnostics.get(reserved_infos.STORYSET_REPEATS.name) self.assertIsInstance(repeats, histogram.GenericSet) self.assertEquals(1, len(repeats)) self.assertEquals(0, list(repeats)[0]) hist = list(results.histograms)[0] trace_start = hist.diagnostics.get(reserved_infos.TRACE_START.name) self.assertIsInstance(trace_start, histogram.DateRange) v_foo = results.FindAllPageSpecificValuesNamed('foo_avg') self.assertEquals(len(v_foo), 1) self.assertEquals(v_foo[0].value, 50) self.assertIsNotNone(v_foo[0].page)