def testBrowserBeforeLaunch(self):
story_set = story.StorySet()
page = page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
name='blank.html')
story_set.AddStory(page)
class TestBeforeLaunch(legacy_page_test.LegacyPageTest):
def __init__(self):
super(TestBeforeLaunch, self).__init__()
self._did_call_will_start = False
self._did_call_did_start = False
def WillStartBrowser(self, platform):
self._did_call_will_start = True
# TODO(simonjam): Test that the profile is available.
def DidStartBrowser(self, browser):
assert self._did_call_will_start
self._did_call_did_start = True
def ValidateAndMeasurePage(self, *_):
assert self._did_call_did_start
test = TestBeforeLaunch()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
options.output_dir = tempdir
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test=test,
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=options.browser_options,
finder_options=options,
results=results,
)
def testSharedPageStateCannotRunOnBrowser(self):
story_set = story.StorySet()
class UnrunnableSharedState(shared_page_state.SharedPageState):
def CanRunOnBrowser(self, browser_info, page):
del browser_info, page # unused
return False
def ValidateAndMeasurePage(self, _):
pass
story_set.AddStory(page_module.Page(
url='file://blank.html', page_set=story_set,
base_dir=util.GetUnittestDataDir(),
shared_page_state_class=UnrunnableSharedState,
name='blank.html'))
class Test(legacy_page_test.LegacyPageTest):
def __init__(self, *args, **kwargs):
super(Test, self).__init__(*args, **kwargs)
self.will_navigate_to_page_called = False
def ValidateAndMeasurePage(self, *args):
del args # unused
raise Exception('Exception should not be thrown')
def WillNavigateToPage(self, page, tab):
del page, tab # unused
self.will_navigate_to_page_called = True
test = Test()
options = options_for_unittests.GetCopy()
SetUpStoryRunnerArguments(options)
# Override defaults from parser creation and arg processing.
options.output_formats = ['none']
options.suppress_gtest_report = True
options.output_dir = None
results = results_options.CreateResults(options)
story_runner.Run(test, story_set, options, results)
self.assertFalse(test.will_navigate_to_page_called)
self.assertEquals(1, len(GetSuccessfulPageRuns(results)))
self.assertTrue(results.had_skips)
self.assertFalse(results.had_failures)
def testAppCrashThenRaiseInTearDownFatal(self):
self.SuppressExceptionFormatting()
story_set = story_module.StorySet()
unit_test_events = [] # track what was called when
class DidRunTestError(Exception):
pass
class TestTearDownSharedState(TestSharedPageState):
def TearDownState(self):
unit_test_events.append('tear-down-state')
raise DidRunTestError
def DumpStateUponFailure(self, story, results):
unit_test_events.append('dump-state')
class Test(legacy_page_test.LegacyPageTest):
def __init__(self, *args):
super(Test, self).__init__(*args)
self.run_count = 0
def RunPage(self, *_):
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 0:
unit_test_events.append('app-crash')
raise exceptions.AppCrashException
def ValidateAndMeasurePage(self, page, tab, results):
pass
story_set.AddStory(DummyLocalStory(TestTearDownSharedState,
name='foo'))
story_set.AddStory(DummyLocalStory(TestTearDownSharedState,
name='bar'))
test = Test()
with self.assertRaises(DidRunTestError):
story_runner.Run(test, story_set, self.options, self.results)
self.assertEqual(['app-crash', 'dump-state', 'tear-down-state'],
unit_test_events)
# The AppCrashException gets added as a failure.
self.assertEquals(1, len(self.results.failures))
def testBrowserStartupURLAndExtraBrowserArgsSetCorrectly(self):
story_set = story_module.StorySet()
google_page = page.Page('http://www.google.com',
startup_url='http://www.google.com',
page_set=story_set,
name='google',
extra_browser_args=['--test_arg1'])
example_page = page.Page('https://www.example.com',
startup_url='https://www.example.com',
page_set=story_set,
name='example',
extra_browser_args=['--test_arg2'])
gmail_page = page.Page('https://www.gmail.com',
startup_url='https://www.gmail.com',
page_set=story_set,
name='gmail',
extra_browser_args=['--test_arg3'])
for p in (google_page, example_page, gmail_page):
story_set.AddStory(p)
shared_state = shared_page_state.SharedPageState(
DummyTest(), self.options, story_set)
try:
for p in (google_page, example_page, gmail_page):
shared_state.WillRunStory(p)
# Fake possible browser saves the browser_options passed into it.
page_level_options = shared_state._possible_browser.browser_options
self.assertEqual(p.startup_url, page_level_options.startup_url)
self.assertNotEqual(
p.startup_url,
shared_state._finder_options.browser_options.startup_url,
'Make sure the startup_url isn\'t polluting the shared page state.'
)
for page_level_argument in p.extra_browser_args:
self.assertIn(page_level_argument,
page_level_options.extra_browser_args)
self.assertNotIn(
page_level_argument, shared_state._finder_options.
browser_options.extra_browser_args,
'Make sure the extra browser arguments aren\'t polluting the '
'shared page state')
finally:
shared_state.TearDownState()
def testGetBrokenExpectations(self):
expectations = ('# tags: [ all ]\n'
'# results: [ Skip ]\n'
'crbug.com/123 [ all ] fake/tw* [ Skip ]\n'
'crbug.com/123 [ all ] fake/one [ Skip ]\n'
'crbug.com/123 [ all ] fake/three [ Skip ]\n'
'crbug.com/123 [ all ] fake1/* [ Skip ]\n')
with mock.patch.object(benchmark.Benchmark,
'Name',
return_value='fake'):
story = mock.MagicMock()
story.name = 'three'
story_set = story_module.StorySet()
story_set._stories.append(story)
b = benchmark.Benchmark()
b.AugmentExpectationsWithFile(expectations)
broken_expectations = b.GetBrokenExpectations(story_set)
self.assertEqual(set(['fake/tw*', 'fake/one']),
broken_expectations)
def testSuccessfulTimelineBasedMeasurementTest(self):
"""Check that PageTest is not required for story_runner.Run.
Any PageTest related calls or attributes need to only be called
for PageTest tests.
"""
class TestSharedTbmState(TestSharedState):
def RunStory(self, results):
pass
test = timeline_based_measurement.TimelineBasedMeasurement(
timeline_based_measurement.Options())
story_set = story_module.StorySet()
story_set.AddStory(DummyLocalStory(TestSharedTbmState))
story_set.AddStory(DummyLocalStory(TestSharedTbmState))
story_set.AddStory(DummyLocalStory(TestSharedTbmState))
story_runner.Run(test, story_set, self.options, self.results)
self.assertEquals(0, len(self.results.failures))
self.assertEquals(3, GetNumberOfSuccessfulPageRuns(self.results))
def testBrowserRestartsAfterEachPage(self):
self.CaptureFormattedException()
story_set = story.StorySet()
story_set.AddStory(page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
name='foo'))
story_set.AddStory(page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
name='bar'))
class Test(legacy_page_test.LegacyPageTest):
def __init__(self, *args, **kwargs):
super(Test, self).__init__(*args, **kwargs)
self.browser_starts = 0
self.platform_name = None
def DidStartBrowser(self, browser):
super(Test, self).DidStartBrowser(browser)
self.browser_starts += 1
self.platform_name = browser.platform.GetOSName()
def ValidateAndMeasurePage(self, page, tab, results):
pass
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
options.output_dir = tempdir
test = Test()
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(options)
story_runner.Run(test, story_set, options, results)
self.assertEquals(len(story_set), len(GetSuccessfulPageRuns(results)))
# Browser is started once per story run, except in ChromeOS where a single
# instance is reused for all stories.
if test.platform_name == 'chromeos':
self.assertEquals(1, test.browser_starts)
else:
self.assertEquals(len(story_set), test.browser_starts)
self.assertFormattedExceptionIsEmpty()
def CreateStorySet(self, options):
"""Makes a PageSet for Dromaeo benchmarks."""
# Subclasses are expected to define class members called query_param and
# tag.
if not hasattr(self, 'query_param') or not hasattr(self, 'tag'):
raise NotImplementedError(
'query_param or tag not in Dromaeo benchmark.')
archive_data_file = '../page_sets/data/dromaeo.%s.json' % self.tag
ps = story.StorySet(archive_data_file=archive_data_file,
base_dir=os.path.dirname(
os.path.abspath(__file__)),
cloud_storage_bucket=story.PUBLIC_BUCKET)
url = 'http://dromaeo.com?%s' % self.query_param
ps.AddStory(
page_module.Page(url,
ps,
ps.base_dir,
make_javascript_deterministic=False))
return ps
def CreateStorySet(self, options):
should_filter_suites = bool(options.suite)
filtered_suite_names = map(
speedometer2_pages.Speedometer2Story.GetFullSuiteName,
speedometer2_pages.Speedometer2Story.GetSuites(options.suite))
ps = story.StorySet(base_dir=_SPEEDOMETER_DIR)
# For a smoke test one iteration is sufficient
if self.enable_smoke_test_mode and not self.iteration_count:
iteration_count = 1
else:
iteration_count = self.iteration_count
ps.AddStory(
speedometer2_pages.Speedometer2Story(ps, should_filter_suites,
filtered_suite_names,
iteration_count))
return ps
def testRunStoryOneDisabledOneNot(self):
story_set = story_module.StorySet()
story_one = DummyLocalStory(TestSharedPageState, name='one')
story_two = DummyLocalStory(TestSharedPageState, name='two')
story_set.AddStory(story_one)
story_set.AddStory(story_two)
results = results_options.CreateResults(
EmptyMetadataForTest(), self.options)
story_runner.Run(_Measurement(), story_set, self.options, results,
expectations=_DisableStoryExpectations(),
metadata=EmptyMetadataForTest())
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
self.assertEquals(2, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(1, GetNumberOfSkippedPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(2, len(values))
def testNoScreenShotTakenForFailedPageDueToNoSupport(self):
self.options.browser_options.take_screenshot_for_failed_page = True
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(page_module.Page('file://blank.html', story_set))
failing_page = FailingTestPage('chrome://version', story_set)
story_set.AddStory(failing_page)
results = results_options.CreateResults(
EmptyMetadataForTest(), self.options)
story_runner.Run(DummyTest(), story_set, self.options, results,
max_failures=2,
metadata=EmptyMetadataForTest())
self.assertEquals(1, len(results.failures))
self.assertEquals(0, len(results.pages_to_profiling_files))
def _RunPageTestThatRaisesAppCrashException(self, test, max_failures):
class TestPage(page_module.Page):
def RunNavigateSteps(self, _):
raise exceptions.AppCrashException
story_set = story.StorySet()
for _ in range(5):
story_set.AddStory(
TestPage('file://blank.html', story_set,
base_dir=util.GetUnittestDataDir()))
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
story_runner.Run(test, story_set, options, results,
max_failures=max_failures)
return results
def testScreenShotTakenForFailedPage(self):
self.CaptureFormattedException()
platform_screenshot_supported = [False]
tab_screenshot_supported = [False]
chrome_version_screen_shot = [None]
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
action_runner.Navigate(self._url)
platform_screenshot_supported[0] = (
action_runner.tab.browser.platform.CanTakeScreenshot)
tab_screenshot_supported[0] = action_runner.tab.screenshot_supported
if not platform_screenshot_supported[0] and tab_screenshot_supported[0]:
chrome_version_screen_shot[0] = action_runner.tab.Screenshot()
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(page_module.Page('file://blank.html', story_set))
failing_page = FailingTestPage('chrome://version', story_set)
story_set.AddStory(failing_page)
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.browser_options.take_screenshot_for_failed_page = True
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
story_runner.Run(DummyTest(), story_set, options, results,
max_failures=2,
metadata=EmptyMetadataForTest())
self.assertEquals(1, len(results.failures))
if not platform_screenshot_supported[0] and tab_screenshot_supported[0]:
self.assertEquals(1, len(results.pages_to_profiling_files))
self.assertIn(failing_page,
results.pages_to_profiling_files)
screenshot_file_path = (
results.pages_to_profiling_files[failing_page][0].GetAbsPath())
try:
actual_screenshot = image_util.FromPngFile(screenshot_file_path)
self.assertEquals(image_util.Pixels(chrome_version_screen_shot[0]),
image_util.Pixels(actual_screenshot))
finally: # Must clean up screenshot file if exists.
os.remove(screenshot_file_path)
def testTranslateScalarNoneValue(self):
story_set = story.StorySet()
p = page.Page('http://www.foo.com/', story_set, name='foo')
scalar_value = {
'type': 'numeric',
'numeric': {
'type': 'scalar',
'unit': 'timeInMs_smallerIsBetter',
'value': None
},
'name': 'foo',
'description': 'desc'
}
v = common_value_helpers.TranslateScalarValue(scalar_value, p)
self.assertIsNone(v.value)
self.assertEquals('Common scalar contained None', v.none_value_reason)
def testEnsureWarmAfterColdRun(self):
with self.captureTrace():
story_set = story.StorySet()
page = page_module.Page('http://google.com',
page_set=story_set,
cache_temperature=cache_temperature.COLD,
name='http://google.com')
cache_temperature.EnsurePageCacheTemperature(page, self._browser)
previous_page = page
page = page_module.Page('http://google.com',
page_set=story_set,
cache_temperature=cache_temperature.WARM,
name='http://google.com')
cache_temperature.EnsurePageCacheTemperature(
page, self._browser, previous_page)
markers = self.traceMarkers()
self.assertNotIn('telemetry.internal.warm_cache.start', markers)
def testScreenShotTakenForFailedPage(self):
self.CaptureFormattedException()
platform_screenshot_supported = [False]
tab_screenshot_supported = [False]
chrome_version_screen_shot = [None]
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
action_runner.Navigate(self._url)
platform_screenshot_supported[0] = (
action_runner.tab.browser.platform.CanTakeScreenshot)
tab_screenshot_supported[
0] = action_runner.tab.screenshot_supported
if not platform_screenshot_supported[
0] and tab_screenshot_supported[0]:
chrome_version_screen_shot[
0] = action_runner.tab.Screenshot()
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(
page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version',
story_set,
name='failing')
story_set.AddStory(failing_page)
self.options.browser_options.take_screenshot_for_failed_page = True
results = RunStorySet(DummyTest(),
story_set,
self.options,
max_failures=2)
self.assertTrue(results.had_failures)
if not platform_screenshot_supported[0] and tab_screenshot_supported[0]:
failed_run = next(run for run in results.IterStoryRuns()
if run.story.name == failing_page.name)
screenshot_file_path = failed_run.GetArtifact(
'screenshot').local_path
actual_screenshot = image_util.FromPngFile(screenshot_file_path)
self.assertEquals(image_util.Pixels(chrome_version_screen_shot[0]),
image_util.Pixels(actual_screenshot))
def testRunStoryAddsDeviceInfo(self):
story_set = story_module.StorySet()
story_set.AddStory(DummyLocalStory(FooStoryState, 'foo', ['bar']))
story_runner.Run(DummyTest(), story_set, self.options, self.results,
metadata=EmptyMetadataForTest())
hs = self.results.histograms
generic_diagnostics = hs.GetSharedDiagnosticsOfType(
histogram_module.GenericSet)
generic_diagnostics_values = [
list(diagnostic) for diagnostic in generic_diagnostics]
self.assertGreater(len(generic_diagnostics), 3)
self.assertIn(['win10'], generic_diagnostics_values)
self.assertIn(['win'], generic_diagnostics_values)
self.assertIn(['amd64'], generic_diagnostics_values)
self.assertIn([8 * (1024 ** 3)], generic_diagnostics_values)
def setUp(self):
story_set = story.StorySet()
story_set.AddStory(
page_module.Page("http://www.foo.com/",
story_set,
name='http://www.foo.com/'))
story_set.AddStory(
page_module.Page("http://www.bar.com/",
story_set,
name='http://www.bar.com/'))
story_set.AddStory(
page_module.Page("http://www.baz.com/",
story_set,
name='http://www.baz.com/'))
self.story_set = story_set
self._output_dir = tempfile.mkdtemp()
self._time_module = mock.patch(
'telemetry.internal.results.page_test_results.time').start()
self._time_module.time.return_value = 0
def testPagesetRepeat(self):
story_set = story.StorySet()
# TODO(eakuefner): Factor this out after flattening page ref in Value
blank_story = DummyLocalUserStory(TestSharedPageState, name='blank')
green_story = DummyLocalUserStory(TestSharedPageState, name='green')
story_set.AddUserStory(blank_story)
story_set.AddUserStory(green_story)
class Measurement(page_test.PageTest):
i = 0
def RunPage(self, page, _, results):
self.i += 1
results.AddValue(
scalar.ScalarValue(page, 'metric', 'unit', self.i))
def ValidateAndMeasurePage(self, page, tab, results):
pass
self.options.page_repeat = 1
self.options.pageset_repeat = 2
self.options.output_formats = []
results = results_options.CreateResults(EmptyMetadataForTest(),
self.options)
story_runner.Run(Measurement(), story_set, self.expectations,
self.options, results)
summary = summary_module.Summary(results.all_page_specific_values)
values = summary.interleaved_computed_per_page_values_and_summaries
blank_value = list_of_scalar_values.ListOfScalarValues(
blank_story, 'metric', 'unit', [1, 3])
green_value = list_of_scalar_values.ListOfScalarValues(
green_story, 'metric', 'unit', [2, 4])
merged_value = list_of_scalar_values.ListOfScalarValues(
None, 'metric', 'unit', [1, 2, 3, 4])
self.assertEquals(4, GetNumberOfSuccessfulPageRuns(results))
self.assertEquals(0, len(results.failures))
self.assertEquals(3, len(values))
self.assertIn(blank_value, values)
self.assertIn(green_value, values)
self.assertIn(merged_value, values)
def testEnsureHotBrowser(self):
with self.CaptureTraceMarkers():
story_set = story.StorySet()
page = page_module.Page(
_TEST_URL,
page_set=story_set,
cache_temperature=cache_temperature.HOT_BROWSER,
name=_TEST_URL)
cache_temperature.EnsurePageCacheTemperature(page, self._browser)
# Browser cache warming happens in a different tab so markers shouldn't
# appear.
self.assertNotIn('telemetry.internal.warm_cache.warm.start',
self.markers)
self.assertNotIn('telemetry.internal.warm_cache.warm.end',
self.markers)
self.assertNotIn('telemetry.internal.warm_cache.hot.start',
self.markers)
self.assertNotIn('telemetry.internal.warm_cache.hot.end', self.markers)
def testRaiseBrowserGoneExceptionFromRestartBrowserBeforeEachPage(self):
self.CaptureFormattedException()
story_set = story.StorySet()
story_set.AddStory(page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
name='foo'))
story_set.AddStory(page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
name='bar'))
story_set.AddStory(page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
name='baz'))
class Test(legacy_page_test.LegacyPageTest):
def __init__(self, *args):
super(Test, self).__init__(
*args, needs_browser_restart_after_each_page=True)
self.run_count = 0
def RestartBrowserBeforeEachPage(self):
# This will only be called twice with 3 pages.
old_run_count = self.run_count
self.run_count += 1
if old_run_count == 1:
raise exceptions.BrowserGoneException(None)
return self._needs_browser_restart_after_each_page
def ValidateAndMeasurePage(self, page, tab, results):
pass
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
test = Test()
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
story_runner.Run(test, story_set, options, results,
metadata=EmptyMetadataForTest())
self.assertEquals(2, test.run_count)
self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
self.assertEquals(1, len(results.failures))
self.assertFormattedExceptionIsEmpty()
def testScreenShotTakenForFailedPage(self):
self.CaptureFormattedException()
platform_screenshot_supported = [False]
tab_screenshot_supported = [False]
chrome_version_screen_shot = [None]
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
action_runner.Navigate(self._url)
platform_screenshot_supported[0] = (
action_runner.tab.browser.platform.CanTakeScreenshot)
tab_screenshot_supported[0] = action_runner.tab.screenshot_supported
if not platform_screenshot_supported[0] and tab_screenshot_supported[0]:
chrome_version_screen_shot[0] = action_runner.tab.Screenshot()
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version', story_set,
name='failing')
story_set.AddStory(failing_page)
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
options = options_for_unittests.GetCopy()
options.output_dir = tempdir
options.output_formats = ['none']
options.browser_options.take_screenshot_for_failed_page = True
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(options)
story_runner.Run(DummyTest(), story_set, options, results,
max_failures=2)
self.assertTrue(results.had_failures)
if not platform_screenshot_supported[0] and tab_screenshot_supported[0]:
failed_run = next(run for run in results._all_page_runs
if run.story.name == failing_page.name)
screenshot_file_path = failed_run.GetArtifact('screenshot').local_path
actual_screenshot = image_util.FromPngFile(screenshot_file_path)
self.assertEquals(image_util.Pixels(chrome_version_screen_shot[0]),
image_util.Pixels(actual_screenshot))
def testTrafficSettings(self):
story_set = story.StorySet()
slow_page = page_module.Page(
'file://green_rect.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='slow',
traffic_setting=traffic_setting_module.GOOD_3G)
fast_page = page_module.Page(
'file://green_rect.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='fast',
traffic_setting=traffic_setting_module.WIFI)
story_set.AddStory(slow_page)
story_set.AddStory(fast_page)
latencies_by_page_in_ms = {}
class MeasureLatency(legacy_page_test.LegacyPageTest):
def __init__(self):
super(MeasureLatency, self).__init__()
self._will_navigate_time = None
def WillNavigateToPage(self, page, tab):
del page, tab # unused
self._will_navigate_time = time.time() * 1000
def ValidateAndMeasurePage(self, page, tab, results):
del results # unused
latencies_by_page_in_ms[page.name] = (time.time() * 1000 -
self._will_navigate_time)
test = MeasureLatency()
results = self.RunStorySet(test, story_set)
self.assertFalse(results.had_failures)
self.assertIn('slow', latencies_by_page_in_ms)
self.assertIn('fast', latencies_by_page_in_ms)
# Slow page should be slower than fast page by at least 40 ms (roundtrip
# time of good 3G) - 2 ms (roundtrip time of Wifi)
self.assertGreater(latencies_by_page_in_ms['slow'],
latencies_by_page_in_ms['fast'] + 40 - 2)
def testBrowserRestartsAfterEachPage(self):
self.CaptureFormattedException()
story_set = story.StorySet()
story_set.AddStory(
page_module.Page('file://blank.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='foo'))
story_set.AddStory(
page_module.Page('file://blank.html',
story_set,
base_dir=util.GetUnittestDataDir(),
name='bar'))
class Test(legacy_page_test.LegacyPageTest):
def __init__(self, *args, **kwargs):
super(Test, self).__init__(*args, **kwargs)
self.browser_starts = 0
def DidStartBrowser(self, *args):
super(Test, self).DidStartBrowser(*args)
self.browser_starts += 1
def ValidateAndMeasurePage(self, page, tab, results):
pass
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
test = Test()
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(),
options)
story_runner.Run(test,
story_set,
options,
results,
tear_down_after_story=True,
metadata=EmptyMetadataForTest())
self.assertEquals(2, len(GetSuccessfulPageRuns(results)))
self.assertEquals(2, test.browser_starts)
self.assertFormattedExceptionIsEmpty()
def testUserAgent(self):
story_set = story.StorySet()
page = page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir(),
shared_page_state_class=shared_page_state.SharedTabletPageState,
name='blank.html')
story_set.AddStory(page)
class TestUserAgent(legacy_page_test.LegacyPageTest):
def ValidateAndMeasurePage(self, page, tab, results):
del page, results # unused
actual_user_agent = tab.EvaluateJavaScript(
'window.navigator.userAgent')
expected_user_agent = user_agent.UA_TYPE_MAPPING['tablet']
assert actual_user_agent.strip() == expected_user_agent
# This is so we can check later that the test actually made it into this
# function. Previously it was timing out before even getting here, which
# should fail, but since it skipped all the asserts, it slipped by.
self.hasRun = True # pylint: disable=attribute-defined-outside-init
test = TestUserAgent()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
with tempfile_ext.NamedTemporaryDirectory('page_E2E_tests') as tempdir:
options.output_dir = tempdir
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
possible_browser = browser_finder.FindBrowser(options)
story_runner.RunStorySet(
test=test,
story_set=story_set,
possible_browser=possible_browser,
expectations=None,
browser_options=options.browser_options,
finder_options=options,
results=results,
)
self.assertTrue(hasattr(test, 'hasRun') and test.hasRun)
def testScreenShotTakenForFailedPageOnSupportedPlatform(self):
fake_platform = self.options.fake_possible_browser.returned_browser.platform
expected_png_base64 = """
iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91
JpzAAAAFklEQVR4Xg3EAQ0AAABAMP1LY3YI7l8l6A
T8tgwbJAAAAABJRU5ErkJggg==
"""
fake_platform.screenshot_png_data = expected_png_base64
self.options.browser_options.take_screenshot_for_failed_page = True
class FailingTestPage(page_module.Page):
def RunNavigateSteps(self, action_runner):
raise exceptions.AppCrashException
story_set = story.StorySet()
story_set.AddStory(page_module.Page('file://blank.html', story_set,
name='blank.html'))
failing_page = FailingTestPage('chrome://version', story_set,
name='failing')
story_set.AddStory(failing_page)
self.options.output_formats = ['json-test-results']
with tempfile_ext.NamedTemporaryDirectory() as tempdir:
self.options.output_dir = tempdir
results = results_options.CreateResults(self.options)
# This ensures the output stream used by the json test results object is
# closed. On windows, we can't delete the temp directory if a file in that
# directory is still in use.
with results:
story_runner.Run(DummyTest(), story_set, self.options, results,
max_failures=2)
self.assertTrue(results.had_failures)
failed_run = next(run for run in results._all_page_runs
if run.story.name == failing_page.name)
screenshot_file_path = failed_run.GetArtifact('screenshot').local_path
actual_screenshot_img = image_util.FromPngFile(screenshot_file_path)
self.assertTrue(
image_util.AreEqual(
image_util.FromBase64Png(expected_png_base64),
actual_screenshot_img))
def testMultipleTabsOkayForMultiTabTest(self):
story_set = story.StorySet()
page = page_module.Page(
'file://blank.html', story_set, base_dir=util.GetUnittestDataDir())
story_set.AddStory(page)
class TestMultiTabs(page_test.PageTest):
def TabForPage(self, _, browser):
return browser.tabs.New()
def ValidateAndMeasurePage(self, _, tab, __):
assert len(tab.browser.tabs) == 2
test = TestMultiTabs()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
story_runner.Run(test, story_set, options, results)
def testEnsureWarmAfterColdRun(self):
with self.CaptureTrace():
story_set = story.StorySet()
page = page_module.Page(
_TEST_URL, page_set=story_set,
cache_temperature=cache_temperature.COLD, name=_TEST_URL)
cache_temperature.EnsurePageCacheTemperature(page, self._browser)
previous_page = page
page = page_module.Page(
_TEST_URL, page_set=story_set,
cache_temperature=cache_temperature.WARM, name=_TEST_URL)
cache_temperature.EnsurePageCacheTemperature(
page, self._browser, previous_page)
markers = self.CollectTraceMarkers()
self.assertNotIn('telemetry.internal.warm_cache.warm.start', markers)
self.assertNotIn('telemetry.internal.warm_cache.warm.end', markers)
self.assertNotIn('telemetry.internal.warm_cache.hot.start', markers)
self.assertNotIn('telemetry.internal.warm_cache.hot.end', markers)
def testTranslateScalarNoneValue(self):
story_set = story.StorySet(base_dir=os.path.dirname(__file__))
p = page.Page('http://www.foo.com/', story_set, story_set.base_dir)
scalar_value = {
'type': 'numeric',
'numeric': {
'type': 'scalar',
'unit': 'timeInMs_smallerIsBetter',
'value': None
},
'grouping_keys': {
'name': 'foo'
}
}
v = common_value_helpers.TranslateScalarValue(scalar_value, p)
self.assertIsNone(v.value)
self.assertEquals('Common scalar contained None', v.none_value_reason)
def testTrafficSettings(self):
story_set = story.StorySet()
slow_page = page_module.Page(
'file://green_rect.html', story_set, base_dir=util.GetUnittestDataDir(),
name='slow',
traffic_setting=traffic_setting_module.REGULAR_2G)
fast_page = page_module.Page(
'file://green_rect.html', story_set, base_dir=util.GetUnittestDataDir(),
name='fast',
traffic_setting=traffic_setting_module.WIFI)
story_set.AddStory(slow_page)
story_set.AddStory(fast_page)
latencies_by_page_in_ms = {}
class MeasureLatency(legacy_page_test.LegacyPageTest):
def __init__(self):
super(MeasureLatency, self).__init__()
self._will_navigate_time = None
def WillNavigateToPage(self, page, tab):
del page, tab # unused
self._will_navigate_time = time.time() * 1000
def ValidateAndMeasurePage(self, page, tab, results):
del results # unused
latencies_by_page_in_ms[page.name] = (
time.time() * 1000 - self._will_navigate_time)
test = MeasureLatency()
options = options_for_unittests.GetCopy()
options.output_formats = ['none']
options.suppress_gtest_report = True
SetUpStoryRunnerArguments(options)
results = results_options.CreateResults(EmptyMetadataForTest(), options)
story_runner.Run(test, story_set, options, results,
metadata=EmptyMetadataForTest())
# Slow page should be slower than fast page by at least 300 ms (roundtrip
# time of 2G) - 2 ms (roundtrip time of Wifi)
self.assertGreater(latencies_by_page_in_ms['slow'],
latencies_by_page_in_ms['fast'] + 300 - 2)
