def RunMeasurement(self, measurement, ps, options=None):
"""Runs a measurement against a pageset, returning the rows its outputs."""
if options is None:
options = options_for_unittests.GetCopy()
assert options
temp_parser = options.CreateParser()
story_runner.AddCommandLineArgs(temp_parser)
defaults = temp_parser.get_default_values()
for k, v in defaults.__dict__.items():
if hasattr(options, k):
continue
setattr(options, k, v)
if isinstance(measurement, legacy_page_test.LegacyPageTest):
measurement.CustomizeBrowserOptions(options.browser_options)
options.output_file = None
options.output_formats = ['none']
options.suppress_gtest_report = True
options.output_trace_tag = None
story_runner.ProcessCommandLineArgs(temp_parser, options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
measurement,
ps,
possible_browser,
None, # expectations
options.browser_options,
options,
results)
return results
def RunPageTest(self, page_test, url, expect_status='PASS'):
"""Run a legacy page_test on a test url and return its measurements.
Args:
page_test: A legacy_page_test.LegacyPageTest instance.
url: A URL for the test page to load, usually a local 'file://' URI to be
served from telemetry/internal/testing. Clients can override the
static method CreateStorySetForTestFile to change this behavior.
expect_status: A string with the expected status of the test run.
Returns:
A dictionary with measurements recorded by the legacy_page_test.
"""
self.assertIsInstance(page_test, legacy_page_test.LegacyPageTest)
page_test.CustomizeBrowserOptions(self.options.browser_options)
story_set = self.CreateStorySetForTest(url)
self.assertEqual(len(story_set), 1)
with results_options.CreateResults(self.options) as results:
story_runner.RunStorySet(page_test, story_set, self.options,
results)
test_results = results_options.ReadTestResults(
self.options.intermediate_dir)
self.assertEqual(len(test_results), 1)
self.test_result = test_results[0]
self.assertEqual(self.test_result['status'], expect_status)
return results_options.ReadMeasurements(self.test_result)
def _RunPageTestThatRaisesAppCrashException(self, test, max_failures):
class TestPage(page_module.Page):
def RunNavigateSteps(self, _):
raise exceptions.AppCrashException
story_set = story.StorySet()
for i in range(5):
story_set.AddStory(
TestPage('file://blank.html', story_set,
base_dir=util.GetUnittestDataDir(), name='foo%d' % i))
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test=test,
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=options.browser_options,
finder_options=options,
results=results,
max_failures=max_failures,
)
return results
def testTrafficSettings(self):
story_set = story.StorySet()
slow_page = page_module.Page(
'file://green_rect.html', story_set, base_dir=util.GetUnittestDataDir(),
name='slow',
traffic_setting=traffic_setting_module.GOOD_3G)
fast_page = page_module.Page(
'file://green_rect.html', story_set, base_dir=util.GetUnittestDataDir(),
name='fast',
traffic_setting=traffic_setting_module.WIFI)
story_set.AddStory(slow_page)
story_set.AddStory(fast_page)
latencies_by_page_in_ms = {}
class MeasureLatency(legacy_page_test.LegacyPageTest):
def __init__(self):
super(MeasureLatency, self).__init__()
self._will_navigate_time = None
def WillNavigateToPage(self, page, tab):
del page, tab # unused
self._will_navigate_time = time.time() * 1000
def ValidateAndMeasurePage(self, page, tab, results):
del results # unused
latencies_by_page_in_ms[page.name] = (
time.time() * 1000 - self._will_navigate_time)
test = MeasureLatency()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
options.output_dir = tempdir
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test=test,
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=options.browser_options,
finder_options=options,
results=results,
)
failure_messages = []
for r in results.all_page_runs:
if r.failure_str:
failure_messages.append(
'Failure message of story %s:\n%s' % (r.story, r.failure_str))
self.assertFalse(results.had_failures, msg=''.join(failure_messages))
self.assertIn('slow', latencies_by_page_in_ms)
self.assertIn('fast', latencies_by_page_in_ms)
# Slow page should be slower than fast page by at least 40 ms (roundtrip
# time of good 3G) - 2 ms (roundtrip time of Wifi)
self.assertGreater(latencies_by_page_in_ms['slow'],
latencies_by_page_in_ms['fast'] + 40 - 2)
def testOneTab(self):
story_set = story.StorySet()
page = page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
name='blank.html')
story_set.AddStory(page)
class TestOneTab(legacy_page_test.LegacyPageTest):
def DidStartBrowser(self, browser):
browser.tabs.New()
def ValidateAndMeasurePage(self, page, tab, results):
del page, results # unused
assert len(tab.browser.tabs) == 1
test = TestOneTab()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
options.output_dir = tempdir
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test=test,
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=options.browser_options,
finder_options=options,
results=results,
)
def testNoScreenShotTakenForFailedPageDueToNoSupport(self):
self.options.browser_options.take_screenshot_for_failed_page = True
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version', story_set,
name='failing')
story_set.AddStory(failing_page)
results = results_options.CreateResults(
EmptyMetadataForTest(), self.options)
possible_browser = browser_finder.FindBrowser(self.options)
story_runner.RunStorySet(
test=DummyTest(),
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=self.options.browser_options,
finder_options=self.options,
results=results,
max_failures=2,
)
self.assertTrue(results.had_failures)
def RunStorySetAndGetResults(self, story_set):
dummy_test = test_stories.DummyStoryTest()
with results_options.CreateResults(self.options) as results:
story_runner.RunStorySet(dummy_test, story_set, self.options, results)
test_results = results_options.ReadIntermediateResults(
self.options.intermediate_dir)['testResults']
self.assertEqual(len(test_results), 1)
return test_results[0]
def Record(self, results):
assert self._story_set.wpr_archive_info, (
'Pageset archive_data_file path must be specified.')
# Always record the benchmark one time only.
self._options.pageset_repeat = 1
self._story_set.wpr_archive_info.AddNewTemporaryRecording()
self._record_page_test.CustomizeBrowserOptions(self._options)
story_runner.RunStorySet(self._record_page_test, self._story_set,
self._options, results)
def testScreenShotTakenForFailedPageOnSupportedPlatform(self):
fake_platform = self.options.fake_possible_browser.returned_browser.platform
expected_png_base64 = """
iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91
JpzAAAAFklEQVR4Xg3EAQ0AAABAMP1LY3YI7l8l6A
T8tgwbJAAAAABJRU5ErkJggg==
"""
fake_platform.screenshot_png_data = expected_png_base64
self.options.browser_options.take_screenshot_for_failed_page = True
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version', story_set,
name='failing')
story_set.AddStory(failing_page)
self.options.output_formats = ['json-test-results']
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
self.options.output_dir = tempdir
results = results_options.CreateResults(
EmptyMetadataForTest(), self.options)
# This ensures the output stream used by the json test results object is
# closed. On windows, we can't delete the temp directory if a file in that
# directory is still in use.
with results:
possible_browser = browser_finder.FindBrowser(self.options)
story_runner.RunStorySet(
test=DummyTest(),
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=self.options.browser_options,
finder_options=self.options,
results=results,
max_failures=2,
)
self.assertTrue(results.had_failures)
artifacts = results._artifact_results.GetTestArtifacts(
failing_page.name)
self.assertIsNotNone(artifacts)
self.assertIn('screenshot', artifacts)
screenshot_file_path = os.path.join(tempdir, artifacts['screenshot'][0])
actual_screenshot_img = image_util.FromPngFile(screenshot_file_path)
self.assertTrue(
image_util.AreEqual(
image_util.FromBase64Png(expected_png_base64),
actual_screenshot_img))
def testScreenShotTakenForFailedPage(self):
self.CaptureFormattedException()
platform_screenshot_supported = [False]
tab_screenshot_supported = [False]
chrome_version_screen_shot = [None]
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
action_runner.Navigate(self._url)
platform_screenshot_supported[0] = (
action_runner.tab.browser.platform.CanTakeScreenshot)
tab_screenshot_supported[0] = action_runner.tab.screenshot_supported
if not platform_screenshot_supported[0] and tab_screenshot_supported[0]:
chrome_version_screen_shot[0] = action_runner.tab.Screenshot()
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version', story_set,
name='failing')
story_set.AddStory(failing_page)
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
options = options_for_unittests.GetCopy()
options.output_dir = tempdir
options.output_formats = ['none']
options.browser_options.take_screenshot_for_failed_page = True
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test=DummyTest(),
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=options.browser_options,
finder_options=options,
results=results,
max_failures=2,
)
self.assertTrue(results.had_failures)
if not platform_screenshot_supported[0] and tab_screenshot_supported[0]:
artifacts = results._artifact_results.GetTestArtifacts(
failing_page.name)
self.assertIsNotNone(artifacts)
self.assertIn('screenshot', artifacts)
screenshot_file_path = os.path.join(tempdir, artifacts['screenshot'][0])
actual_screenshot = image_util.FromPngFile(screenshot_file_path)
self.assertEquals(image_util.Pixels(chrome_version_screen_shot[0]),
image_util.Pixels(actual_screenshot))
def testSharedPageStateCannotRunOnBrowser(self):
story_set = story.StorySet()
class UnrunnableSharedState(shared_page_state.SharedPageState):
def CanRunOnBrowser(self, browser_info, page):
del browser_info, page # unused
return False
def ValidateAndMeasurePage(self, _):
pass
story_set.AddStory(page_module.Page(
url='file://blank.html', page_set=story_set,
base_dir=util.GetUnittestDataDir(),
shared_page_state_class=UnrunnableSharedState,
name='blank.html'))
class Test(legacy_page_test.LegacyPageTest):
def __init__(self, *args, **kwargs):
super(Test, self).__init__(*args, **kwargs)
self.will_navigate_to_page_called = False
def ValidateAndMeasurePage(self, *args):
del args # unused
raise Exception('Exception should not be thrown')
def WillNavigateToPage(self, page, tab):
del page, tab # unused
self.will_navigate_to_page_called = True
test = Test()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test=test,
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=options.browser_options,
finder_options=options,
results=results,
)
self.assertFalse(test.will_navigate_to_page_called)
self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
self.assertEquals(1, len(results.skipped_values))
self.assertFalse(results.had_failures)
def testBrowserRestartsAfterEachPage(self):
self.CaptureFormattedException()
story_set = story.StorySet()
story_set.AddStory(page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
name='foo'))
story_set.AddStory(page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
name='bar'))
class Test(legacy_page_test.LegacyPageTest):
def __init__(self, *args, **kwargs):
super(Test, self).__init__(*args, **kwargs)
self.browser_starts = 0
self.platform_name = None
def DidStartBrowser(self, browser):
super(Test, self).DidStartBrowser(browser)
self.browser_starts += 1
self.platform_name = browser.platform.GetOSName()
def ValidateAndMeasurePage(self, page, tab, results):
pass
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
options.output_dir = tempdir
test = Test()
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test,
story_set,
possible_browser,
None, # expectations
options.browser_options,
options,
results)
self.assertEquals(len(story_set), len(GetSuccessfulPageRuns(results)))
# Browser is started once per story run, except in ChromeOS where a single
# instance is reused for all stories.
if test.platform_name == 'chromeos':
self.assertEquals(1, test.browser_starts)
else:
self.assertEquals(len(story_set), test.browser_starts)
self.assertFormattedExceptionIsEmpty()
def RunMeasurement(self, measurement, story_set, run_options):
"""Runs a measurement against a story set, returning a results object.
Args:
measurement: A test object: either a story_test.StoryTest or
legacy_page_test.LegacyPageTest instance.
story_set: A story set.
run_options: An object with all options needed to run stories; can be
created with the help of options_for_unittests.GetRunOptions().
"""
if isinstance(measurement, legacy_page_test.LegacyPageTest):
measurement.CustomizeBrowserOptions(run_options.browser_options)
with results_options.CreateResults(
run_options, benchmark_name=BENCHMARK_NAME) as results:
story_runner.RunStorySet(measurement, story_set, run_options,
results)
return results
def Record(self, results):
assert self._story_set.wpr_archive_info, (
'Pageset archive_data_file path must be specified.')
# Always record the benchmark one time only.
self._options.pageset_repeat = 1
self._story_set.wpr_archive_info.AddNewTemporaryRecording()
self._record_page_test.CustomizeBrowserOptions(self._options)
possible_browser = browser_finder.FindBrowser(self._options)
story_runner.RunStorySet(
test=self._record_page_test,
story_set=self._story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=self._options.browser_options,
finder_options=self._options,
results=results,
)
def testBrowserBeforeLaunch(self):
story_set = story.StorySet()
page = page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
name='blank.html')
story_set.AddStory(page)
class TestBeforeLaunch(legacy_page_test.LegacyPageTest):
def __init__(self):
super(TestBeforeLaunch, self).__init__()
self._did_call_will_start = False
self._did_call_did_start = False
def WillStartBrowser(self, platform):
self._did_call_will_start = True
# TODO(simonjam): Test that the profile is available.
def DidStartBrowser(self, browser):
assert self._did_call_will_start
self._did_call_did_start = True
def ValidateAndMeasurePage(self, *_):
assert self._did_call_did_start
test = TestBeforeLaunch()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
options.output_dir = tempdir
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test=test,
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=options.browser_options,
finder_options=options,
results=results,
)
def testUserAgent(self):
story_set = story.StorySet()
page = page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
shared_page_state_class=shared_page_state.SharedTabletPageState,
name='blank.html')
story_set.AddStory(page)
class TestUserAgent(legacy_page_test.LegacyPageTest):
def ValidateAndMeasurePage(self, page, tab, results):
del page, results # unused
actual_user_agent = tab.EvaluateJavaScript(
'window.navigator.userAgent')
expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
assert actual_user_agent.strip() == expected_user_agent
# This is so we can check later that the test actually made it into this
# function. Previously it was timing out before even getting here, which
# should fail, but since it skipped all the asserts, it slipped by.
self.hasRun = True # pylint: disable=attribute-defined-outside-init
test = TestUserAgent()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
options.output_dir = tempdir
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test=test,
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=options.browser_options,
finder_options=options,
results=results,
)
self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
def testCleanUpPage(self):
story_set = story.StorySet()
page = page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
name='blank.html')
story_set.AddStory(page)
class Test(legacy_page_test.LegacyPageTest):
def __init__(self):
super(Test, self).__init__()
self.did_call_clean_up = False
def ValidateAndMeasurePage(self, *_):
raise legacy_page_test.Failure
def DidRunPage(self, platform):
del platform # unused
self.did_call_clean_up = True
test = Test()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
options.output_dir = tempdir
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test=test,
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=options.browser_options,
finder_options=options,
results=results,
)
assert test.did_call_clean_up
def testWebPageReplay(self):
story_set = example_domain.ExampleDomainPageSet()
body = []
class TestWpr(legacy_page_test.LegacyPageTest):
def ValidateAndMeasurePage(self, page, tab, results):
del page, results # unused
body.append(tab.EvaluateJavaScript('document.body.innerText'))
def DidRunPage(self, platform):
# Force the replay server to restart between pages; this verifies that
# the restart mechanism works.
platform.network_controller.StopReplay()
test = TestWpr()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test=test,
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=options.browser_options,
finder_options=options,
results=results,
)
self.longMessage = True
self.assertIn('Example Domain', body[0],
msg='URL: %s' % story_set.stories[0].url)
self.assertIn('Example Domain', body[1],
msg='URL: %s' % story_set.stories[1].url)
self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
self.assertFalse(results.had_failures)
def RunStories(self, stories, **kwargs):
story_set = test_stories.DummyStorySet(stories)
with results_options.CreateResults(
self.options, benchmark_name='benchmark') as results:
story_runner.RunStorySet(self.mock_story_test, story_set,
self.options, results, **kwargs)
def RunStorySet(test, story_set, options, **kwargs):
with results_options.CreateResults(options) as results:
story_runner.RunStorySet(test, story_set, options, results, **kwargs)
return results
def testPageResetWhenBrowserReusedBetweenStories(self):
class NoClosingBrowserSharedState(shared_page_state.SharedPageState):
# Simulate what ChromeOS does.
def ShouldStopBrowserAfterStoryRun(self, s):
del s # unused
return False
# Loads a page and scrolls it to the end.
class ScrollingPage(page_module.Page):
def __init__(self, url, page_set, base_dir):
super(ScrollingPage, self).__init__(page_set=page_set,
base_dir=base_dir,
shared_page_state_class=
NoClosingBrowserSharedState,
url=url, name='ScrollingPage')
def RunPageInteractions(self, action_runner):
action_runner.ScrollPage()
# Loads same page as ScrollingPage() and records if the scroll position is
# at the top of the page (in was_page_at_top_on_start).
class CheckScrollPositionPage(page_module.Page):
def __init__(self, url, page_set, base_dir):
super(CheckScrollPositionPage, self).__init__(
page_set=page_set, base_dir=base_dir,
shared_page_state_class=NoClosingBrowserSharedState, url=url,
name='CheckScroll')
self.was_page_at_top_on_start = False
def RunPageInteractions(self, action_runner):
scroll_y = action_runner.tab.EvaluateJavaScript('window.scrollY')
self.was_page_at_top_on_start = scroll_y == 0
class Test(legacy_page_test.LegacyPageTest):
def ValidateAndMeasurePage(self, *_):
pass
story_set = story.StorySet()
story_set.AddStory(ScrollingPage(
url='file://page_with_swipeables.html', page_set=story_set,
base_dir=util.GetUnittestDataDir()))
test_page = CheckScrollPositionPage(
url='file://page_with_swipeables.html', page_set=story_set,
base_dir=util.GetUnittestDataDir())
story_set.AddStory(test_page)
test = Test()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test=test,
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=options.browser_options,
finder_options=options,
results=results,
)
self.assertTrue(test_page.was_page_at_top_on_start)
