def testUnknownExceptionIsFatal(self):
self.SuppressExceptionFormatting()
story_set = story_module.StorySet()
class UnknownException(Exception):
pass
# This erroneous test is set up to raise exception for the 2nd story
# run.
class Test(page_test.PageTest):
def __init__(self, *args):
super(Test, self).__init__(*args)
self.run_count = 0
def RunPage(self, *_):
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 1:
raise UnknownException('FooBarzException')
def ValidateAndMeasurePage(self, page, tab, results):
pass
s1 = DummyLocalStory(TestSharedPageState)
s2 = DummyLocalStory(TestSharedPageState)
story_set.AddStory(s1)
story_set.AddStory(s2)
test = Test()
with self.assertRaises(UnknownException):
story_runner.Run(
test, story_set, self.options, self.results)
self.assertEqual(set([s2]), self.results.pages_that_failed)
self.assertEqual(set([s1]), self.results.pages_that_succeeded)
self.assertIn('FooBarzException', self.fake_stdout.getvalue())
def testNoProfilingFilesCreatedForPageByDefault(self):
self.CaptureFormattedException()
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
action_runner.Navigate(self._url)
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(page_module.Page('file://blank.html', story_set))
failing_page = FailingTestPage('chrome://version', story_set)
story_set.AddStory(failing_page)
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(DummyTest(),
story_set,
options,
results,
max_failures=2)
self.assertEquals(1, len(results.failures))
self.assertEquals(0, len(results.pages_to_profiling_files))
def RunStoryTest(self, s, expected_successes):
test = DummyTest()
story_runner.Run(
test, s, self.options, self.results)
self.assertEquals(0, len(self.results.failures))
self.assertEquals(expected_successes,
GetNumberOfSuccessfulPageRuns(self.results))
def testWebPageReplay(self):
story_set = example_domain.ExampleDomainPageSet()
body = []
class TestWpr(legacy_page_test.LegacyPageTest):
def ValidateAndMeasurePage(self, page, tab, results):
del page, results # unused
body.append(tab.EvaluateJavaScript2('document.body.innerText'))
def DidRunPage(self, platform):
# Force the replay server to restart between pages; this verifies that
# the restart mechanism works.
platform.network_controller.StopReplay()
test = TestWpr()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(test, story_set, options, results)
self.longMessage = True
self.assertIn('Example Domain',
body[0],
msg='URL: %s' % story_set.stories[0].url)
self.assertIn('Example Domain',
body[1],
msg='URL: %s' % story_set.stories[1].url)
self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
self.assertEquals(0, len(results.failures))
def testRunPageWithProfilingFlag(self):
story_set = story.StorySet()
story_set.AddStory(
page_module.Page('file://blank.html',
story_set,
base_dir=util.GetUnittestDataDir()))
class Measurement(legacy_page_test.LegacyPageTest):
def ValidateAndMeasurePage(self, page, tab, results):
pass
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
options.reset_results = None
options.upload_results = None
options.results_label = None
options.output_dir = tempfile.mkdtemp()
options.profiler = 'trace'
try:
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(Measurement(), story_set, options, results)
self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
self.assertEquals(0, len(results.failures))
self.assertEquals(0, len(results.all_page_specific_values))
self.assertTrue(
os.path.isfile(
os.path.join(options.output_dir, 'blank_html.html')))
finally:
shutil.rmtree(options.output_dir)
def testCleanUpPage(self):
story_set = story.StorySet()
page = page_module.Page('file://blank.html',
story_set,
base_dir=util.GetUnittestDataDir())
story_set.AddStory(page)
class Test(legacy_page_test.LegacyPageTest):
def __init__(self):
super(Test, self).__init__()
self.did_call_clean_up = False
def ValidateAndMeasurePage(self, *_):
raise legacy_page_test.Failure
def DidRunPage(self, platform):
del platform # unused
self.did_call_clean_up = True
test = Test()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(test, story_set, options, results)
assert test.did_call_clean_up
def testBrowserBeforeLaunch(self):
story_set = story.StorySet()
page = page_module.Page('file://blank.html',
story_set,
base_dir=util.GetUnittestDataDir())
story_set.AddStory(page)
class TestBeforeLaunch(legacy_page_test.LegacyPageTest):
def __init__(self):
super(TestBeforeLaunch, self).__init__()
self._did_call_will_start = False
self._did_call_did_start = False
def WillStartBrowser(self, platform):
self._did_call_will_start = True
# TODO(simonjam): Test that the profile is available.
def DidStartBrowser(self, browser):
assert self._did_call_will_start
self._did_call_did_start = True
def ValidateAndMeasurePage(self, *_):
assert self._did_call_did_start
test = TestBeforeLaunch()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(test, story_set, options, results)
def testUserAgent(self):
story_set = story.StorySet()
page = page_module.Page(
'file://blank.html',
story_set,
base_dir=util.GetUnittestDataDir(),
shared_page_state_class=shared_page_state.SharedTabletPageState)
story_set.AddStory(page)
class TestUserAgent(legacy_page_test.LegacyPageTest):
def ValidateAndMeasurePage(self, page, tab, results):
del page, results # unused
actual_user_agent = tab.EvaluateJavaScript2(
'window.navigator.userAgent')
expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
assert actual_user_agent.strip() == expected_user_agent
# This is so we can check later that the test actually made it into this
# function. Previously it was timing out before even getting here, which
# should fail, but since it skipped all the asserts, it slipped by.
self.hasRun = True # pylint: disable=attribute-defined-outside-init
test = TestUserAgent()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(test, story_set, options, results)
self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
def testRunStoryWithLongURLPage(self):
story_set = story_module.StorySet()
story_set.AddStory(page_module.Page('file://long' + 'g' * 180,
story_set, name='test'))
test = DummyTest()
story_runner.Run(test, story_set,
self.options, self.results,
metadata=EmptyMetadataForTest())
def Record(self, results):
assert self._page_set.wpr_archive_info, (
'Pageset archive_data_file path must be specified.')
self._page_set.wpr_archive_info.AddNewTemporaryRecording()
self._record_page_test.CustomizeBrowserOptions(self._options)
story_runner.Run(self._record_page_test, self._page_set,
test_expectations.TestExpectations(), self._options,
results)
def _testMaxFailuresOptionIsRespectedAndOverridable(
self, num_failing_user_stories, runner_max_failures, options_max_failures,
expected_num_failures):
class SimpleSharedState(
shared_state.SharedState):
_fake_platform = FakePlatform()
_current_user_story = None
@property
def platform(self):
return self._fake_platform
def WillRunUserStory(self, story):
self._current_user_story = story
def RunUserStory(self, results):
self._current_user_story.Run()
def DidRunUserStory(self, results):
pass
def GetTestExpectationAndSkipValue(self, expectations):
return 'pass', None
def TearDownState(self):
pass
class FailingUserStory(user_story.UserStory):
def __init__(self):
super(FailingUserStory, self).__init__(
shared_state_class=SimpleSharedState,
is_local=True)
self.was_run = False
def Run(self):
self.was_run = True
raise page_test.Failure
self.SuppressExceptionFormatting()
uss = user_story_set.UserStorySet()
for _ in range(num_failing_user_stories):
uss.AddUserStory(FailingUserStory())
options = _GetOptionForUnittest()
options.output_formats = ['none']
options.suppress_gtest_report = True
if options_max_failures:
options.max_failures = options_max_failures
results = results_options.CreateResults(EmptyMetadataForTest(), options)
story_runner.Run(
DummyTest(), uss, test_expectations.TestExpectations(), options,
results, max_failures=runner_max_failures)
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(expected_num_failures, len(results.failures))
for ii, story in enumerate(uss.user_stories):
self.assertEqual(story.was_run, ii < expected_num_failures)
def testTrafficSettings(self):
story_set = story.StorySet()
slow_page = page_module.Page(
'file://green_rect.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='slow',
traffic_setting=traffic_setting_module.GOOD_3G)
fast_page = page_module.Page(
'file://green_rect.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='fast',
traffic_setting=traffic_setting_module.WIFI)
story_set.AddStory(slow_page)
story_set.AddStory(fast_page)
latencies_by_page_in_ms = {}
class MeasureLatency(legacy_page_test.LegacyPageTest):
def __init__(self):
super(MeasureLatency, self).__init__()
self._will_navigate_time = None
def WillNavigateToPage(self, page, tab):
del page, tab # unused
self._will_navigate_time = time.time() * 1000
def ValidateAndMeasurePage(self, page, tab, results):
del results # unused
latencies_by_page_in_ms[page.name] = (time.time() * 1000 -
self._will_navigate_time)
test = MeasureLatency()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
options.output_dir = tempdir
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(test, story_set, options, results)
failure_messages = []
for r in results.all_page_runs:
if r.failure_str:
failure_messages.append(
'Failure message of story %s:\n%s' %
(r.story, r.failure_str))
self.assertFalse(results.had_failures,
msg=''.join(failure_messages))
self.assertIn('slow', latencies_by_page_in_ms)
self.assertIn('fast', latencies_by_page_in_ms)
# Slow page should be slower than fast page by at least 40 ms (roundtrip
# time of good 3G) - 2 ms (roundtrip time of Wifi)
self.assertGreater(latencies_by_page_in_ms['slow'],
latencies_by_page_in_ms['fast'] + 40 - 2)
def testScreenShotTakenForFailedPage(self):
self.CaptureFormattedException()
platform_screenshot_supported = [False]
tab_screenshot_supported = [False]
chrome_version_screen_shot = [None]
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
action_runner.Navigate(self._url)
platform_screenshot_supported[0] = (
action_runner.tab.browser.platform.CanTakeScreenshot)
tab_screenshot_supported[
0] = action_runner.tab.screenshot_supported
if not platform_screenshot_supported[
0] and tab_screenshot_supported[0]:
chrome_version_screen_shot[
0] = action_runner.tab.Screenshot()
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(
page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version',
story_set,
name='failing')
story_set.AddStory(failing_page)
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
options = options_for_unittests.GetCopy()
options.output_dir = tempdir
options.output_formats = ['none']
options.browser_options.take_screenshot_for_failed_page = True
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(DummyTest(),
story_set,
options,
results,
max_failures=2,
metadata=EmptyMetadataForTest())
self.assertTrue(results.had_failures)
if not platform_screenshot_supported[
0] and tab_screenshot_supported[0]:
artifacts = results._artifact_results.GetTestArtifacts(
failing_page.name)
self.assertIsNotNone(artifacts)
self.assertIn('screenshot', artifacts)
screenshot_file_path = os.path.join(tempdir,
artifacts['screenshot'][0])
actual_screenshot = image_util.FromPngFile(
screenshot_file_path)
self.assertEquals(
image_util.Pixels(chrome_version_screen_shot[0]),
image_util.Pixels(actual_screenshot))
def testPagesetRepeat(self):
story_set = story_module.StorySet()
# TODO(eakuefner): Factor this out after flattening page ref in Value
blank_story = DummyLocalStory(TestSharedPageState, name='blank')
green_story = DummyLocalStory(TestSharedPageState, name='green')
story_set.AddStory(blank_story)
story_set.AddStory(green_story)
class Measurement(legacy_page_test.LegacyPageTest):
i = 0
def RunPage(self, page, _, results):
self.i += 1
results.AddValue(
scalar.ScalarValue(
page,
'metric',
'unit',
self.i,
improvement_direction=improvement_direction.UP))
def ValidateAndMeasurePage(self, page, tab, results):
pass
self.options.page_repeat = 1
self.options.pageset_repeat = 2
self.options.output_formats = []
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
story_runner.Run(Measurement(), story_set, self.options, results)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
blank_value = list_of_scalar_values.ListOfScalarValues(
blank_story,
'metric',
'unit', [1, 3],
improvement_direction=improvement_direction.UP)
green_value = list_of_scalar_values.ListOfScalarValues(
green_story,
'metric',
'unit', [2, 4],
improvement_direction=improvement_direction.UP)
merged_value = list_of_scalar_values.ListOfScalarValues(
None,
'metric',
'unit',
[1, 3, 2, 4],
std=math.sqrt(2), # Pooled standard deviation.
improvement_direction=improvement_direction.UP)
self.assertEquals(4, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(3, len(values))
self.assertIn(blank_value, values)
self.assertIn(green_value, values)
self.assertIn(merged_value, values)
def Record(self, results):
assert self._story_set.wpr_archive_info, (
'Pageset archive_data_file path must be specified.')
self._story_set.wpr_archive_info.AddNewTemporaryRecording()
self._record_page_test.CustomizeBrowserOptions(self._options)
story_runner.Run(self._record_page_test,
self._story_set,
self._options,
results,
metadata=self._CreateBenchmarkMetadata())
def Record(self, results):
assert self._story_set.wpr_archive_info, (
'Pageset archive_data_file path must be specified.')
# Always record the benchmark one time only.
self._options.pageset_repeat = 1
self._story_set.wpr_archive_info.AddNewTemporaryRecording()
self._record_page_test.CustomizeBrowserOptions(self._options)
story_runner.Run(self._record_page_test, self._story_set,
self._options, results)
def testPageResetWhenBrowserReusedBetweenStories(self):
class NoClosingBrowserSharedState(shared_page_state.SharedPageState):
# Simulate what ChromeOS does.
def ShouldStopBrowserAfterStoryRun(self, s):
del s # unused
return False
# Loads a page and scrolls it to the end.
class ScrollingPage(page_module.Page):
def __init__(self, url, page_set, base_dir):
super(ScrollingPage, self).__init__(page_set=page_set,
base_dir=base_dir,
shared_page_state_class=
NoClosingBrowserSharedState,
url=url, name='ScrollingPage')
def RunPageInteractions(self, action_runner):
action_runner.ScrollPage()
# Loads same page as ScrollingPage() and records if the scroll position is
# at the top of the page (in was_page_at_top_on_start).
class CheckScrollPositionPage(page_module.Page):
def __init__(self, url, page_set, base_dir):
super(CheckScrollPositionPage, self).__init__(
page_set=page_set, base_dir=base_dir,
shared_page_state_class=NoClosingBrowserSharedState, url=url,
name='CheckScroll')
self.was_page_at_top_on_start = False
def RunPageInteractions(self, action_runner):
scroll_y = action_runner.tab.EvaluateJavaScript('window.scrollY')
self.was_page_at_top_on_start = scroll_y == 0
class Test(legacy_page_test.LegacyPageTest):
def ValidateAndMeasurePage(self, *_):
pass
story_set = story.StorySet()
story_set.AddStory(ScrollingPage(
url='file://page_with_swipeables.html', page_set=story_set,
base_dir=util.GetUnittestDataDir()))
test_page = CheckScrollPositionPage(
url='file://page_with_swipeables.html', page_set=story_set,
base_dir=util.GetUnittestDataDir())
story_set.AddStory(test_page)
test = Test()
options = options_for_unittests.GetCopy()
SetUpStoryRunnerArguments(options)
# Override defaults from parser creation and arg processing.
options.output_formats = ['none']
options.suppress_gtest_report = True
options.output_dir = None
results = results_options.CreateResults(options)
story_runner.Run(test, story_set, options, results)
self.assertTrue(test_page.was_page_at_top_on_start)
def testScreenShotTakenForFailedPage(self):
self.CaptureFormattedException()
platform_screenshot_supported = [False]
tab_screenshot_supported = [False]
chrome_version_screen_shot = [None]
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
action_runner.Navigate(self._url)
platform_screenshot_supported[0] = (
action_runner.tab.browser.platform.CanTakeScreenshot)
tab_screenshot_supported[
0] = action_runner.tab.screenshot_supported
if not platform_screenshot_supported[
0] and tab_screenshot_supported[0]:
chrome_version_screen_shot[
0] = action_runner.tab.Screenshot()
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(
page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version',
story_set,
name='failing')
story_set.AddStory(failing_page)
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.browser_options.take_screenshot_for_failed_page = True
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(DummyTest(),
story_set,
options,
results,
max_failures=2,
metadata=EmptyMetadataForTest())
self.assertEquals(1, len(results.failures))
if not platform_screenshot_supported[0] and tab_screenshot_supported[0]:
self.assertEquals(1, len(results.pages_to_profiling_files))
self.assertIn(failing_page, results.pages_to_profiling_files)
screenshot_file_path = (
results.pages_to_profiling_files[failing_page][0].GetAbsPath())
try:
actual_screenshot = image_util.FromPngFile(
screenshot_file_path)
self.assertEquals(
image_util.Pixels(chrome_version_screen_shot[0]),
image_util.Pixels(actual_screenshot))
finally: # Must clean up screenshot file if exists.
os.remove(screenshot_file_path)
def testScreenShotTakenForFailedPageOnSupportedPlatform(self):
fake_platform = self.options.fake_possible_browser.returned_browser.platform
expected_png_base64 = """
iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91
JpzAAAAFklEQVR4Xg3EAQ0AAABAMP1LY3YI7l8l6A
T8tgwbJAAAAABJRU5ErkJggg==
"""
fake_platform.screenshot_png_data = expected_png_base64
self.options.browser_options.take_screenshot_for_failed_page = True
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(
page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version',
story_set,
name='failing')
story_set.AddStory(failing_page)
self.options.output_formats = ['json-test-results']
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
self.options.output_dir = tempdir
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
# This ensures the output stream used by the json test results object is
# closed. On windows, we can't delete the temp directory if a file in that
# directory is still in use.
with results:
story_runner.Run(DummyTest(),
story_set,
self.options,
results,
max_failures=2,
metadata=EmptyMetadataForTest())
self.assertTrue(results.had_failures)
artifacts = results._artifact_results.GetTestArtifacts(
failing_page.name)
self.assertIsNotNone(artifacts)
self.assertIn('screenshot', artifacts)
screenshot_file_path = os.path.join(tempdir,
artifacts['screenshot'][0])
actual_screenshot_img = image_util.FromPngFile(
screenshot_file_path)
self.assertTrue(
image_util.AreEqual(
image_util.FromBase64Png(expected_png_base64),
actual_screenshot_img))
def testExceptionRaisedInSharedStateTearDown(self):
self.SuppressExceptionFormatting()
story_set = story_module.StorySet()
class SharedStoryThatCausesAppCrash(TestSharedPageState):
def TearDownState(self):
raise TestOnlyException()
story_set.AddStory(DummyLocalStory(SharedStoryThatCausesAppCrash))
with self.assertRaises(TestOnlyException):
story_runner.Run(DummyTest(), story_set, self.options,
self.results)
def testRunStorySet(self):
number_stories = 3
story_set = story_module.StorySet()
for i in xrange(number_stories):
story_set.AddStory(DummyLocalStory(FooStoryState, name='story_%d' % i))
test = DummyTest()
story_runner.Run(
test, story_set, self.options, self.results,
metadata=EmptyMetadataForTest())
self.assertEquals(0, len(self.results.failures))
self.assertEquals(number_stories,
GetNumberOfSuccessfulPageRuns(self.results))
def testRaiseBrowserGoneExceptionFromRestartBrowserBeforeEachPage(self):
self.CaptureFormattedException()
story_set = story.StorySet()
story_set.AddStory(
page_module.Page('file://blank.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='foo'))
story_set.AddStory(
page_module.Page('file://blank.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='bar'))
story_set.AddStory(
page_module.Page('file://blank.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='baz'))
class Test(legacy_page_test.LegacyPageTest):
def __init__(self, *args):
super(Test, self).__init__(
*args, needs_browser_restart_after_each_page=True)
self.run_count = 0
def RestartBrowserBeforeEachPage(self):
# This will only be called twice with 3 pages.
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 1:
raise exceptions.BrowserGoneException(None)
return self._needs_browser_restart_after_each_page
def ValidateAndMeasurePage(self, page, tab, results):
pass
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
test = Test()
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(test,
story_set,
options,
results,
metadata=EmptyMetadataForTest())
self.assertEquals(2, test.run_count)
self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
self.assertEquals(1, len(results.failures))
self.assertFormattedExceptionIsEmpty()
def testBrowserRestartsAfterEachPage(self):
self.CaptureFormattedException()
story_set = story.StorySet()
story_set.AddStory(
page_module.Page('file://blank.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='foo'))
story_set.AddStory(
page_module.Page('file://blank.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='bar'))
class Test(legacy_page_test.LegacyPageTest):
def __init__(self, *args, **kwargs):
super(Test, self).__init__(*args, **kwargs)
self.browser_starts = 0
self.platform_name = None
def DidStartBrowser(self, browser):
super(Test, self).DidStartBrowser(browser)
self.browser_starts += 1
self.platform_name = browser.platform.GetOSName()
def ValidateAndMeasurePage(self, page, tab, results):
pass
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
options.output_dir = tempdir
test = Test()
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(test,
story_set,
options,
results,
metadata=EmptyMetadataForTest())
self.assertEquals(len(story_set),
len(GetSuccessfulPageRuns(results)))
# Browser is started once per story run, except in ChromeOS where a single
# instance is reused for all stories.
if test.platform_name == 'chromeos':
self.assertEquals(1, test.browser_starts)
else:
self.assertEquals(len(story_set), test.browser_starts)
self.assertFormattedExceptionIsEmpty()
def runCredentialsTest(self, credentials_backend):
story_set = story.StorySet()
did_run = [False]
try:
class TestSharedState(shared_page_state.SharedPageState):
def ShouldStopBrowserAfterStoryRun(self, _):
# Do not close browser for LoginNoLongerNeeded to get called.
return False
with tempfile.NamedTemporaryFile(delete=False) as f:
page = page_module.Page(
'file://blank.html',
story_set,
base_dir=util.GetUnittestDataDir(),
shared_page_state_class=TestSharedState,
credentials_path=f.name,
name='blank.html')
page.credentials = "test"
story_set.AddStory(page)
f.write(SIMPLE_CREDENTIALS_STRING)
class TestThatInstallsCredentialsBackend(
legacy_page_test.LegacyPageTest):
def __init__(self, credentials_backend):
super(TestThatInstallsCredentialsBackend, self).__init__()
self._credentials_backend = credentials_backend
def DidStartBrowser(self, browser):
browser.credentials.AddBackend(self._credentials_backend)
def ValidateAndMeasurePage(self, *_):
did_run[0] = True
test = TestThatInstallsCredentialsBackend(credentials_backend)
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(test,
story_set,
options,
results,
metadata=EmptyMetadataForTest())
finally:
os.remove(f.name)
return did_run[0]
def testTearDownStateAfterEachStoryOrStorySetRun(self):
class TestSharedStateForTearDown(TestSharedState):
num_of_tear_downs = 0
def RunStory(self, results):
pass
def TearDownState(self):
TestSharedStateForTearDown.num_of_tear_downs += 1
story_set = story_module.StorySet()
story_set.AddStory(
DummyLocalStory(TestSharedStateForTearDown, name='foo'))
story_set.AddStory(
DummyLocalStory(TestSharedStateForTearDown, name='bar'))
story_set.AddStory(
DummyLocalStory(TestSharedStateForTearDown, name='baz'))
TestSharedStateForTearDown.num_of_tear_downs = 0
story_runner.Run(mock.MagicMock(), story_set, self.options,
self.results)
self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 1)
TestSharedStateForTearDown.num_of_tear_downs = 0
story_runner.Run(mock.MagicMock(),
story_set,
self.options,
self.results,
tear_down_after_story=True)
self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 3)
self.options.pageset_repeat = 5
TestSharedStateForTearDown.num_of_tear_downs = 0
story_runner.Run(mock.MagicMock(),
story_set,
self.options,
self.results,
tear_down_after_story_set=True)
self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 5)
def testAppCrashExceptionCausesFailureValue(self):
self.SuppressExceptionFormatting()
us = user_story_set.UserStorySet()
class SharedUserStoryThatCausesAppCrash(TestSharedPageState):
def WillRunUserStory(self, user_storyz):
raise exceptions.AppCrashException('App Foo crashes')
us.AddUserStory(DummyLocalUserStory(SharedUserStoryThatCausesAppCrash))
story_runner.Run(
DummyTest(), us, self.expectations, self.options, self.results)
self.assertEquals(1, len(self.results.failures))
self.assertEquals(0, GetNumberOfSuccessfulPageRuns(self.results))
self.assertIn('App Foo crashes', self.fake_stdout.getvalue())
def run_benchmark(benchmark, finder_options):
benchmark.CustomizeBrowserOptions(finder_options.browser_options)
pt = benchmark.CreatePageTest(finder_options)
pt.__name__ = benchmark.__class__.__name__
stories = benchmark.CreateStorySet(finder_options)
benchmark_metadata = benchmark.GetMetadata()
with results_options.CreateResults(
benchmark_metadata, finder_options,
benchmark.ValueCanBeAddedPredicate) as results:
failures = story_runner.Run(pt, stories, finder_options, results,
benchmark.max_failures)
results._SerializeTracesToDirPath(results._output_dir)
return failures
def testTearDownStateAfterEachStoryRun(self):
class TestSharedStateForTearDown(TestSharedState):
num_of_tear_downs = 0
def RunStory(self, results):
pass
def TearDownState(self):
TestSharedStateForTearDown.num_of_tear_downs += 1
story_set = story_module.StorySet()
story_set.AddStory(DummyLocalStory(TestSharedStateForTearDown))
story_set.AddStory(DummyLocalStory(TestSharedStateForTearDown))
story_set.AddStory(DummyLocalStory(TestSharedStateForTearDown))
TestSharedStateForTearDown.num_of_tear_downs = 0
story_runner.Run(mock.MagicMock(), story_set, self.options, self.results)
self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 1)
TestSharedStateForTearDown.num_of_tear_downs = 0
story_runner.Run(mock.MagicMock(), story_set, self.options, self.results,
should_tear_down_state_after_each_story_run=True)
self.assertEquals(TestSharedStateForTearDown.num_of_tear_downs, 3)
def Run(self, finder_options):
"""Run this test with the given options.
Returns:
The number of failure values (up to 254) or 255 if there is an uncaught
exception.
"""
self.CustomizeBrowserOptions(finder_options.browser_options)
pt = self.CreatePageTest(finder_options)
pt.__name__ = self.__class__.__name__
if hasattr(self, '_disabled_strings'):
# pylint: disable=protected-access
pt._disabled_strings = self._disabled_strings
if hasattr(self, '_enabled_strings'):
# pylint: disable=protected-access
pt._enabled_strings = self._enabled_strings
expectations = self.CreateExpectations()
us = self.CreateStorySet(finder_options)
if isinstance(pt, page_test.PageTest):
if any(not isinstance(p, page.Page) for p in us.user_stories):
raise Exception(
'PageTest must be used with StorySet containing only '
'telemetry.page.Page user stories.')
benchmark_metadata = self.GetMetadata()
with results_options.CreateResults(
benchmark_metadata, finder_options,
self.ValueCanBeAddedPredicate) as results:
try:
story_runner.Run(pt,
us,
expectations,
finder_options,
results,
max_failures=self._max_failures)
return_code = min(254, len(results.failures))
except Exception:
exception_formatter.PrintFormattedException()
return_code = 255
bucket = cloud_storage.BUCKET_ALIASES[finder_options.upload_bucket]
if finder_options.upload_results:
results.UploadTraceFilesToCloud(bucket)
results.UploadProfilingFilesToCloud(bucket)
results.PrintSummary()
return return_code
def testTrafficSettings(self):
story_set = story.StorySet()
slow_page = page_module.Page(
'file://green_rect.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='slow',
traffic_setting=traffic_setting_module.REGULAR_2G)
fast_page = page_module.Page(
'file://green_rect.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='fast',
traffic_setting=traffic_setting_module.WIFI)
story_set.AddStory(slow_page)
story_set.AddStory(fast_page)
latencies_by_page_in_ms = {}
class MeasureLatency(legacy_page_test.LegacyPageTest):
def __init__(self):
super(MeasureLatency, self).__init__()
self._will_navigate_time = None
def WillNavigateToPage(self, page, tab):
del page, tab # unused
self._will_navigate_time = time.time() * 1000
def ValidateAndMeasurePage(self, page, tab, results):
del results # unused
latencies_by_page_in_ms[page.name] = (time.time() * 1000 -
self._will_navigate_time)
test = MeasureLatency()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(test,
story_set,
options,
results,
metadata=EmptyMetadataForTest())
# Slow page should be slower than fast page by at least 300 ms (roundtrip
# time of 2G) - 2 ms (roundtrip time of Wifi)
self.assertGreater(latencies_by_page_in_ms['slow'],
latencies_by_page_in_ms['fast'] + 300 - 2)