def testTraceCaptureUponFailure(self):
ps = self.CreateEmptyPageSet()
ps.AddStory(FailedTimelinebasedMeasurementPage(ps, ps.base_dir))
options = tbm_module.Options()
options.config.enable_chrome_trace = True
options.SetTimelineBasedMetrics(['sampleMetric'])
tbm = tbm_module.TimelineBasedMeasurement(options)
results = self.RunMeasurement(tbm, ps, self._options)
self.assertTrue(results.had_failures)
self.assertEquals(1, len(results.FindAllTraceValues()))
def testSmoothnessTimelineBasedMeasurementForSmoke(self):
ps = self.CreatePageSetFromFileInUnittestDataDir(
'interaction_enabled_page.html')
setattr(ps.pages[0], 'RunSmoothness', {
'action': 'wait',
'javascript': 'window.animationDone'
})
measurement = tbm_module.TimelineBasedMeasurement()
results = self.RunMeasurement(measurement, ps, options=self._options)
self.assertEquals(0, len(results.failures))
v = results.FindAllPageSpecificValuesNamed('CenterAnimation-jank')
self.assertEquals(len(v), 1)
v = results.FindAllPageSpecificValuesNamed('DrawerAnimation-jank')
self.assertEquals(len(v), 1)
def testTimelineBasedMeasurementGestureAdjustmentSmoke(self):
ps = self.CreateEmptyPageSet()
ps.AddStory(TestTimelinebasedMeasurementPage(
ps, ps.base_dir, trigger_scroll_gesture=True))
options = tbm_module.Options()
options.SetLegacyTimelineBasedMetrics([smoothness.SmoothnessMetric()])
tbm = tbm_module.TimelineBasedMeasurement(options)
results = self.RunMeasurement(tbm, ps, options=self._options)
self.assertEquals(0, len(results.failures))
v = results.FindAllPageSpecificValuesFromIRNamed(
'Gesture_Scroll', 'frame_time_discrepancy')
self.assertEquals(len(v), 1)
def testTimelineBasedMeasurementGestureAdjustmentSmoke(self):
ps = self.CreateEmptyPageSet()
ps.AddUserStory(
TestTimelinebasedMeasurementPage(ps,
ps.base_dir,
trigger_scroll_gesture=True))
tbm = tbm_module.TimelineBasedMeasurement(tbm_module.Options())
measurement = tbpt_module.TimelineBasedPageTest(tbm)
results = self.RunMeasurement(measurement, ps, options=self._options)
self.assertEquals(0, len(results.failures))
v = results.FindAllPageSpecificValuesNamed(
'Gesture_Scroll-frame_time_discrepancy')
self.assertEquals(len(v), 1)
def WillNavigateToPage(self, page, tab):
tracing_controller = tab.browser.platform.tracing_controller
# FIXME: Remove webkit.console when blink.console lands in chromium and
# the ref builds are updated. crbug.com/386847
custom_categories = [
'webkit.console', 'blink.console', 'benchmark',
'trace_event_overhead'
]
category_filter = tracing_category_filter.TracingCategoryFilter(
','.join(custom_categories))
self._tbm = timeline_based_measurement.TimelineBasedMeasurement(
timeline_based_measurement.Options(category_filter),
_CustomResultsWrapper)
self._tbm.WillRunUserStory(tracing_controller,
page.GetSyntheticDelayCategories())
def testSmoothnessTimelineBasedMeasurementForSmoke(self):
ps = self.CreateEmptyPageSet()
ps.AddStory(TestTimelinebasedMeasurementPage(
ps, ps.base_dir, trigger_animation=True))
tbm = tbm_module.TimelineBasedMeasurement(tbm_module.Options())
results = self.RunMeasurement(tbm, ps, options=self._options)
self.assertEquals(0, len(results.failures))
v = results.FindAllPageSpecificValuesFromIRNamed(
'CenterAnimation', 'frame_time_discrepancy')
self.assertEquals(len(v), 1)
v = results.FindAllPageSpecificValuesFromIRNamed(
'DrawerAnimation', 'frame_time_discrepancy')
self.assertEquals(len(v), 1)
def testSmoothnessTimelineBasedMeasurementForSmoke(self):
ps = self.CreateEmptyPageSet()
ps.AddPage(
TestTimelinebasedMeasurementPage(ps,
ps.base_dir,
trigger_animation=True))
measurement = tbm_module.TimelineBasedMeasurement()
results = self.RunMeasurement(measurement, ps, options=self._options)
self.assertEquals(0, len(results.failures))
v = results.FindAllPageSpecificValuesNamed('CenterAnimation-jank')
self.assertEquals(len(v), 1)
v = results.FindAllPageSpecificValuesNamed('DrawerAnimation-jank')
self.assertEquals(len(v), 1)
def WillNavigateToPage(self, page, tab):
# FIXME: Remove webkit.console when blink.console lands in chromium and
# the ref builds are updated. crbug.com/386847
custom_categories = [
'webkit.console', 'blink.console', 'benchmark',
'trace_event_overhead'
]
category_filter = chrome_trace_category_filter.ChromeTraceCategoryFilter(
','.join(custom_categories))
options = timeline_based_measurement.Options(category_filter)
options.config.enable_platform_display_trace = True
options.SetLegacyTimelineBasedMetrics([smoothness.SmoothnessMetric()])
self._tbm = timeline_based_measurement.TimelineBasedMeasurement(
options, self._results_wrapper)
self._tbm.WillRunStory(tab.browser.platform)
def testTraceCaptureUponFailure(self):
ps = self.CreateEmptyPageSet()
ps.AddStory(FailedTimelinebasedMeasurementPage(ps, ps.base_dir))
options = tbm_module.Options()
options.config.enable_chrome_trace = True
options.SetTimelineBasedMetrics(['sampleMetric'])
tbm = tbm_module.TimelineBasedMeasurement(options)
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
self._options.output_dir = tempdir
results = self.RunMeasurement(tbm, ps, self._options)
self.assertTrue(results.had_failures)
runs = list(results.IterRunsWithTraces())
self.assertEquals(1, len(runs))
def testTraceCaptureUponFailure(self):
story_set = self.CreateEmptyPageSet()
story_set.AddStory(
FailedTimelinebasedMeasurementPage(story_set, story_set.base_dir))
options = tbm_module.Options()
options.config.enable_chrome_trace = True
options.SetTimelineBasedMetrics(['sampleMetric'])
tbm = tbm_module.TimelineBasedMeasurement(options)
results = self.RunMeasurement(tbm,
story_set,
run_options=self._options)
self.assertTrue(results.had_failures)
runs = list(results.IterRunsWithTraces())
self.assertEquals(1, len(runs))
def testTBM2ForSmoke(self):
ps = self.CreateEmptyPageSet()
ps.AddStory(TestTimelinebasedMeasurementPage(ps, ps.base_dir))
options = tbm_module.Options()
options.SetTimelineBasedMetrics(['sample_metric.html'])
tbm = tbm_module.TimelineBasedMeasurement(options)
results = self.RunMeasurement(tbm, ps, self._options)
self.assertEquals(0, len(results.failures))
v_foo = results.FindAllPageSpecificValuesNamed('foo')
v_bar = results.FindAllPageSpecificValuesNamed('bar')
self.assertEquals(len(v_foo), 1)
self.assertEquals(len(v_bar), 1)
self.assertEquals(v_foo[0].value, 1)
self.assertEquals(v_bar[0].value, 2)
def WillNavigateToPage(self, page, tab):
# FIXME: Remove webkit.console when blink.console lands in chromium and
# the ref builds are updated. crbug.com/386847
custom_categories = [
'webkit.console', 'blink.console', 'benchmark',
'trace_event_overhead'
]
category_filter = tracing_category_filter.TracingCategoryFilter(
','.join(custom_categories))
options = timeline_based_measurement.Options(category_filter)
options.SetLegacyTimelineBasedMetrics([smoothness.SmoothnessMetric()])
for delay in page.GetSyntheticDelayCategories():
options.category_filter.AddSyntheticDelay(delay)
self._tbm = timeline_based_measurement.TimelineBasedMeasurement(
options, self._results_wrapper)
self._tbm.WillRunStory(tab.browser.platform)
def testSuccessfulTimelineBasedMeasurementTest(self):
"""Check that PageTest is not required for story_runner.Run.
Any PageTest related calls or attributes need to only be called
for PageTest tests.
"""
class TestSharedTbmState(TestSharedState):
def RunStory(self, results):
pass
test = timeline_based_measurement.TimelineBasedMeasurement(
timeline_based_measurement.Options())
story_set = story_module.StorySet()
story_set.AddStory(DummyLocalStory(TestSharedTbmState))
story_set.AddStory(DummyLocalStory(TestSharedTbmState))
story_set.AddStory(DummyLocalStory(TestSharedTbmState))
story_runner.Run(test, story_set, self.options, self.results)
self.assertEquals(0, len(self.results.failures))
self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results))
def testTBM2ForSmoke(self):
ps = self.CreateEmptyPageSet()
ps.AddStory(
TestTimelinebasedMeasurementPage(
ps, ps.base_dir, additional_metrics=['sampleMetric']))
options = tbm_module.Options()
options.config.enable_chrome_trace = True
tbm = tbm_module.TimelineBasedMeasurement(options)
results = self.RunMeasurement(tbm, ps, self._options)
self.assertFalse(results.had_failures)
tbm_metrics = []
for story_run in results.IterStoryRuns():
tbm_metrics += story_run.tbm_metrics
self.assertEqual(tbm_metrics, ['sampleMetric'])
histogram_dicts = results.AsHistogramDicts()
hs = histogram_set.HistogramSet()
hs.ImportDicts(histogram_dicts)
self.assertEquals(4, len(hs))
hist = hs.GetFirstHistogram()
benchmarks = hist.diagnostics.get(reserved_infos.BENCHMARKS.name)
self.assertIsInstance(benchmarks, generic_set.GenericSet)
self.assertEquals(1, len(benchmarks))
self.assertEquals(page_test_test_case.BENCHMARK_NAME,
list(benchmarks)[0])
stories = hist.diagnostics.get(reserved_infos.STORIES.name)
self.assertIsInstance(stories, generic_set.GenericSet)
self.assertEquals(1, len(stories))
self.assertEquals('interaction_enabled_page.html', list(stories)[0])
repeats = hist.diagnostics.get(reserved_infos.STORYSET_REPEATS.name)
self.assertIsInstance(repeats, generic_set.GenericSet)
self.assertEquals(1, len(repeats))
self.assertEquals(0, list(repeats)[0])
hist = hs.GetFirstHistogram()
trace_start = hist.diagnostics.get(reserved_infos.TRACE_START.name)
self.assertIsInstance(trace_start, date_range.DateRange)
v_foo = results.FindAllPageSpecificValuesNamed('foo_avg')
self.assertEquals(len(v_foo), 1)
self.assertEquals(v_foo[0].value, 50)
self.assertIsNotNone(v_foo[0].page)
def testGPUTimesTimelineBasedMeasurementForSmoke(self):
ps = self.CreateEmptyPageSet()
ps.AddStory(TestTimelinebasedMeasurementPage(
ps, ps.base_dir, trigger_animation=True))
cat_filter = tracing_category_filter.TracingCategoryFilter(
'disabled-by-default-gpu.service')
tbm_option = tbm_module.Options(overhead_level=cat_filter)
tbm = tbm_module.TimelineBasedMeasurement(tbm_option)
results = self.RunMeasurement(tbm, ps, options=self._options)
self.assertEquals(0, len(results.failures))
v = results.FindAllPageSpecificValuesFromIRNamed(
'CenterAnimation', 'browser_compositor_max_cpu_time')
self.assertEquals(len(v), 1)
self.assertGreater(v[0].value, 0)
v = results.FindAllPageSpecificValuesFromIRNamed(
'DrawerAnimation', 'browser_compositor_max_cpu_time')
self.assertEquals(len(v), 1)
self.assertGreater(v[0].value, 0)
def testSuccessfulTimelineBasedMeasurementTest(self):
"""Check that PageTest is not required for story_runner.Run.
Any PageTest related calls or attributes need to only be called
for PageTest tests.
"""
class TestSharedTbmState(TestSharedState):
def RunStory(self, results):
pass
TEST_WILL_RUN_STORY = 'test.WillRunStory'
TEST_MEASURE = 'test.Measure'
TEST_DID_RUN_STORY = 'test.DidRunStory'
EXPECTED_CALLS_IN_ORDER = [TEST_WILL_RUN_STORY,
TEST_MEASURE,
TEST_DID_RUN_STORY]
test = timeline_based_measurement.TimelineBasedMeasurement(
timeline_based_measurement.Options())
manager = mock.MagicMock()
test.WillRunStory = mock.MagicMock()
test.Measure = mock.MagicMock()
test.DidRunStory = mock.MagicMock()
manager.attach_mock(test.WillRunStory, TEST_WILL_RUN_STORY)
manager.attach_mock(test.Measure, TEST_MEASURE)
manager.attach_mock(test.DidRunStory, TEST_DID_RUN_STORY)
story_set = story_module.StorySet()
story_set.AddStory(DummyLocalStory(TestSharedTbmState, name='foo'))
story_set.AddStory(DummyLocalStory(TestSharedTbmState, name='bar'))
story_set.AddStory(DummyLocalStory(TestSharedTbmState, name='baz'))
story_runner.Run(
test, story_set, self.options, self.results,
metadata=EmptyMetadataForTest())
self.assertEquals(0, len(self.results.failures))
self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results))
self.assertEquals(3*EXPECTED_CALLS_IN_ORDER,
[call[0] for call in manager.mock_calls])
def testMainthreadJankTimelineBasedMeasurement(self):
ps = self.CreateEmptyPageSet()
ps.AddStory(TestTimelinebasedMeasurementPage(
ps, ps.base_dir, trigger_jank=True))
tbm = tbm_module.TimelineBasedMeasurement(tbm_module.Options())
results = self.RunMeasurement(tbm, ps, options=self._options)
self.assertEquals(0, len(results.failures))
# In interaction_enabled_page.html, we create a jank loop based on
# window.performance.now() (basically loop for x milliseconds).
# Since window.performance.now() uses wall-time instead of thread time,
# we only assert the biggest jank > 50ms here to account for the fact
# that the browser may deschedule during the jank loop.
v = results.FindAllPageSpecificValuesFromIRNamed(
'JankThreadJSRun', 'responsive-biggest_jank_thread_time')
self.assertGreaterEqual(v[0].value, 50)
v = results.FindAllPageSpecificValuesFromIRNamed(
'JankThreadJSRun', 'responsive-total_big_jank_thread_time')
self.assertGreaterEqual(v[0].value, 50)
def testMainthreadJankTimelineBasedMeasurement(self):
ps = self.CreateEmptyPageSet()
ps.AddPage(TestTimelinebasedMeasurementPage(ps, ps.base_dir))
measurement = tbm_module.TimelineBasedMeasurement()
results = self.RunMeasurement(measurement, ps, options=self._options)
self.assertEquals(0, len(results.failures))
# In interaction_enabled_page.html, we create a jank loop based on
# window.performance.now() (basically loop for x milliseconds).
# Since window.performance.now() uses wall-time
# instead of thread time, we set time to looping to 100ms in
# interaction_enabled_page.html and only assert the biggest jank > 50ms here
# to account for the fact that the browser may deschedule during the jank
# loop.
v = results.FindAllPageSpecificValuesNamed(
'JankThreadJSRun-responsive-biggest_jank_thread_time')
self.assertGreaterEqual(v[0].value, 50)
v = results.FindAllPageSpecificValuesNamed(
'JankThreadJSRun-responsive-total_big_jank_thread_time')
self.assertGreaterEqual(v[0].value, 50)
def testTBM2ForSmoke(self):
ps = self.CreateEmptyPageSet()
ps.AddStory(TestTimelinebasedMeasurementPage(ps, ps.base_dir))
options = tbm_module.Options()
options.config.enable_chrome_trace = True
options.SetTimelineBasedMetric('sampleMetric')
tbm = tbm_module.TimelineBasedMeasurement(options)
results = self.RunMeasurement(tbm, ps, self._options)
self.assertEquals(0, len(results.failures))
self.assertEquals(9, len(results.value_set))
self.assertEquals(1, len(results.value_set[0]['diagnostics']))
iter_info = results.value_set[0]['diagnostics']['iteration']
self.assertEqual('IterationInfo', iter_info['type'])
self.assertEqual('', iter_info['benchmarkName'])
self.assertEqual('interaction_enabled_page.html',
iter_info['storyDisplayName'])
self.assertEqual({}, iter_info['storyGroupingKeys'])
self.assertEqual(0, iter_info['storyRepeatCounter'])
self.assertEqual(0, iter_info['storysetRepeatCounter'])
self.assertEqual('file://interaction_enabled_page.html',
iter_info['storyUrl'])
v_foo = results.FindAllPageSpecificValuesNamed('foo')
v_bar = results.FindAllPageSpecificValuesNamed('bar')
v_baz_avg = results.FindAllPageSpecificValuesNamed('baz_avg')
v_baz_sum = results.FindAllPageSpecificValuesNamed('baz_sum')
v_baz_count = results.FindAllPageSpecificValuesNamed('baz_count')
self.assertEquals(len(v_foo), 1)
self.assertEquals(len(v_bar), 1)
self.assertEquals(v_foo[0].value, 1)
self.assertIsNotNone(v_foo[0].page)
self.assertEquals(v_bar[0].value, 2)
self.assertIsNotNone(v_bar[0].page)
self.assertEquals(len(v_baz_avg), 1)
self.assertEquals(len(v_baz_sum), 1)
self.assertEquals(len(v_baz_count), 1)
def testFastTimelineBasedMeasurementForSmoke(self):
ps = self.CreateEmptyPageSet()
ps.AddUserStory(
TestTimelinebasedMeasurementPage(ps,
ps.base_dir,
trigger_slow=True))
tbm = tbm_module.TimelineBasedMeasurement(tbm_module.Options())
measurement = tbm_module.TimelineBasedPageTest(tbm)
results = self.RunMeasurement(measurement, ps, options=self._options)
self.assertEquals([], results.failures)
expected_names = set([
'SlowThreadJsRun-fast-duration',
'SlowThreadJsRun-fast-idle_time',
'SlowThreadJsRun-fast-incremental_marking',
'SlowThreadJsRun-fast-incremental_marking_outside_idle',
'SlowThreadJsRun-fast-mark_compactor',
'SlowThreadJsRun-fast-mark_compactor_outside_idle',
'SlowThreadJsRun-fast-scavenger',
'SlowThreadJsRun-fast-scavenger_outside_idle',
'SlowThreadJsRun-fast-total_garbage_collection',
'SlowThreadJsRun-fast-total_garbage_collection_outside_idle',
'trace',
])
if platform.GetHostPlatform().GetOSName() != 'win':
# CPU metric is only supported non-Windows platforms.
expected_names.add('SlowThreadJsRun-fast-cpu_time')
self.assertEquals(
expected_names,
set(v.name for v in results.all_page_specific_values))
# In interaction_enabled_page.html, the "slow" interaction executes
# a loop with window.performance.now() to wait 200ms.
# fast-duration measures wall time so its value should be at least 200ms.
v = results.FindAllPageSpecificValuesNamed(
'SlowThreadJsRun-fast-duration')
self.assertGreaterEqual(v[0].value, 200.0)
def testTBM2ForSmoke(self):
ps = self.CreateEmptyPageSet()
ps.AddStory(TestTimelinebasedMeasurementPage(ps, ps.base_dir))
options = tbm_module.Options()
options.config.enable_chrome_trace = True
options.SetTimelineBasedMetrics(['sampleMetric'])
tbm = tbm_module.TimelineBasedMeasurement(options)
results = self.RunMeasurement(tbm, ps, self._options)
self.assertEquals(0, len(results.failures))
self.assertEquals(1, len(results.histograms))
foos = results.histograms.GetHistogramsNamed('foo')
self.assertEquals(1, len(foos))
hist = foos[0]
benchmarks = hist.diagnostics.get(reserved_infos.BENCHMARKS.name)
self.assertIsInstance(benchmarks, histogram.GenericSet)
self.assertEquals(1, len(benchmarks))
self.assertEquals('', list(benchmarks)[0])
stories = hist.diagnostics.get(reserved_infos.STORIES.name)
self.assertIsInstance(stories, histogram.GenericSet)
self.assertEquals(1, len(stories))
self.assertEquals('interaction_enabled_page.html', list(stories)[0])
repeats = hist.diagnostics.get(reserved_infos.STORYSET_REPEATS.name)
self.assertIsInstance(repeats, histogram.GenericSet)
self.assertEquals(1, len(repeats))
self.assertEquals(0, list(repeats)[0])
hist = list(results.histograms)[0]
trace_start = hist.diagnostics.get(reserved_infos.TRACE_START.name)
self.assertIsInstance(trace_start, histogram.DateRange)
v_foo = results.FindAllPageSpecificValuesNamed('foo_avg')
self.assertEquals(len(v_foo), 1)
self.assertEquals(v_foo[0].value, 50)
self.assertIsNotNone(v_foo[0].page)
